From 3c6d9f1a4a43353d72f27593322069266c21bb00 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:57:53 +0000 Subject: [PATCH 01/74] ramp: keep DeepSeek-V4 FP8 residents as their source bytes (opt-in mxfp8) bind_deepseek_v4_flash_non_vq_weights(resident_format="mxfp8") binds every FP8 resident that lands on a plain nn.Linear, or on wo_a's MultiLinear, as its shipped e4m3 bytes and ue8m0 exponents in a weights-only mxfp8 module (nn.QuantizedLinear, or mlx-lm's QuantizedMultiLinear for wo_a) instead of decoding it to bf16. Every weight dequantizes bit-identically to the bf16 decode, at 8.25 instead of 16 bits per weight. Activations are not quantized. The default stays "bf16", so nothing changes until the Mac kill test passes. load_authenticated_dsv4_composite passes the option through, and the report lists the repacked tensors and the .scales they created, without counting the scales as logical tensors. keep.convert.fp8_block.repack_fp8_block_as_mxfp8 does the byte copy and refuses anything that would need rounding: scales that are not exact powers of two, a NaN scale byte, or a 32-wide group that would straddle two source blocks. Building the same weights through mx.quantize(mode= "mxfp8") is not exact (it changed 608 of 98,304 values in a check), which is why the copy goes through this function. benchmarks/bench_dsv4_mxfp8_residents.py is the kill test: bit-exactness against today's decode, then M=1 latency against bf16 and affine-8 g64, with wo_a timed in its grouped form. Kill if any bit differs, if mxfp8 is under 1.5x bf16, or if it is over 1.1x affine-8. On CPU MLX 0.31.2 the exactness half passes at all eight V4-Flash resident shapes; the speed half has to run on the Mac. Estimate from the roadmap's byte table: on V4-Flash this would shrink residents from about 15.7 GB to about 9.9 GB in memory and cut decode reads by a similar amount. --- benchmarks/bench_dsv4_mxfp8_residents.py | 415 +++++++++++++++++++ src/keep/convert/fp8_block.py | 125 +++++- src/ramp/models/deepseek_v4_flash_adapter.py | 191 ++++++++- src/ramp/models/dsv4_composite_loader.py | 16 +- tests/test_bench_dsv4_mxfp8_residents.py | 168 ++++++++ tests/test_deepseek_v4_flash_adapter.py | 139 +++++++ tests/test_fp8_block_dequant.py | 142 +++++++ 7 files changed, 1189 insertions(+), 7 deletions(-) create mode 100644 benchmarks/bench_dsv4_mxfp8_residents.py create mode 100644 tests/test_bench_dsv4_mxfp8_residents.py diff --git a/benchmarks/bench_dsv4_mxfp8_residents.py b/benchmarks/bench_dsv4_mxfp8_residents.py new file mode 100644 index 0000000..cbfcae5 --- /dev/null +++ b/benchmarks/bench_dsv4_mxfp8_residents.py @@ -0,0 +1,415 @@ +"""Kill test: keep DeepSeek-V4 FP8 residents as their source bytes (mxfp8). + +Two checks in one fresh process: + +1. **Exactness.** Each chosen resident, byte-copied into MLX ``mxfp8`` by + :func:`keep.convert.fp8_block.repack_fp8_block_as_mxfp8`, must dequantize + bit-identically to :func:`keep.convert.fp8_block.dequantize_fp8_block`, + which is what RAMP binds in bf16 today. +2. **Decode-shape speed (M=1).** Time ``quantized_matmul(mode="mxfp8")`` + against the bf16 matmul RAMP runs today and an affine-8 g64 baseline (the + roadmap's S1 alternative). + +Verdict, from the ranked plan: **KILL** if any bit differs, if mxfp8 is less +than 1.5x the bf16 speed, or if it is more than 10% slower than affine-8. +Speed is judged on the summed per-call medians across the chosen tensors, the +way one decode step reads them. + +Inputs: a resident package or the source checkpoint (``--checkpoint``, a +directory with ``model.safetensors.index.json``), or ``--synthetic`` FP8 +tensors at the real DeepSeek-V4-Flash shapes, which needs no download. +Weights are cycled across ``--copies`` buffers so each timed call reads DRAM, +not a cache. Speed only means something on an Apple GPU; on a CPU device the +script still runs the exactness half and marks the speed rows invalid. + +Example (M4 Max, no model download needed):: + + uv run python benchmarks/bench_dsv4_mxfp8_residents.py --synthetic \\ + --output-json artifacts/benchmarks/dsv4-mxfp8-residents-m4max.json +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import os +import platform +import statistics +import sys +import time +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any + +import mlx.core as mx +import numpy as np + +from keep.convert.fp8_block import dequantize_fp8_block, repack_fp8_block_as_mxfp8 +from keep.io.source_safetensors import read_safetensors_file_header + +REPO_ROOT = Path(__file__).resolve().parents[1] +HEAVY_JOB_LOCK = REPO_ROOT / ".keep-heavy-job.lock" +RECORD_TYPE = "dsv4_mxfp8_resident_kill_test_v1" +FP8_BLOCK = (128, 128) + +#: DeepSeek-V4-Flash-0731 resident shapes (out, in) as they ship, from the +#: model defaults: hidden 4096, 64 heads x 512, q_lora_rank 1024, o_groups 8 x +#: o_lora_rank 1024, shared expert width 2048. ``wo_a`` ships flat and runs as +#: an 8-group MultiLinear; see :data:`GROUPED_TENSORS`. +V4_FLASH_RESIDENT_SHAPES: dict[str, tuple[int, int]] = { + "attn.wq_a": (1024, 4096), + "attn.wq_b": (32768, 1024), + "attn.wkv": (512, 4096), + "attn.wo_a": (8192, 4096), + "attn.wo_b": (4096, 8192), + "ffn.shared_experts.w1": (2048, 4096), + "ffn.shared_experts.w2": (4096, 2048), + "ffn.shared_experts.w3": (2048, 4096), +} + +#: Tensors timed as a grouped (batched) matmul, the way RAMP calls them: +#: ``wo_a`` is ``(o_groups * o_lora_rank, in)`` on disk and +#: ``(o_groups, o_lora_rank, in)`` at runtime, fed one row per group. +GROUPED_TENSORS: dict[str, int] = {"attn.wo_a": 8} + +MIN_SPEEDUP_OVER_BF16 = 1.5 +MAX_SLOWDOWN_VS_AFFINE8 = 1.10 +_FORBIDDEN_ENV = ("GLM_MLX_WIRED_LIMIT_GB", "GLM_SINGLE_HOST_MLX_WIRED_LIMIT_GB") + + +def synthetic_fp8_pair( + rng: np.random.Generator, shape: tuple[int, int] +) -> tuple[np.ndarray, np.ndarray]: + """Random finite e4m3 codes plus ue8m0 block exponents at ``shape``.""" + + rows, cols = shape + codes = rng.integers(0, 256, size=(rows, cols), dtype=np.uint8) + codes = np.where((codes & 0x7F) == 0x7F, 0x7E, codes).astype(np.uint8) + blocks = (-(-rows // FP8_BLOCK[0]), -(-cols // FP8_BLOCK[1])) + scales = rng.integers(115, 126, size=blocks, dtype=np.uint8) + return codes, scales + + +class _CheckpointReader: + """Raw-byte reader over a safetensors index; one memmap per shard.""" + + def __init__(self, root: Path) -> None: + index = json.loads((root / "model.safetensors.index.json").read_text()) + self.root = root + self.weight_map: dict[str, str] = dict(index["weight_map"]) + self._shards: dict[str, tuple[np.memmap, int, Mapping[str, Any]]] = {} + + def raw(self, name: str) -> tuple[np.ndarray, str, tuple[int, ...]]: + shard = self.weight_map[name] + if shard not in self._shards: + path = self.root / shard + header = read_safetensors_file_header(path) + self._shards[shard] = ( + np.memmap(path, dtype=np.uint8, mode="r"), + header.header_size, + header.tensors, + ) + mm, header_size, tensors = self._shards[shard] + tensor = tensors[name] + start, end = tensor.data_offsets + base = 8 + header_size + return np.array(mm[base + start : base + end]), tensor.dtype, tuple(tensor.shape) + + def fp8_pair(self, prefix: str) -> tuple[np.ndarray, np.ndarray]: + codes, code_dtype, shape = self.raw(f"{prefix}.weight") + scales, scale_dtype, scale_shape = self.raw(f"{prefix}.scale") + if code_dtype != "F8_E4M3" or scale_dtype != "F8_E8M0": + raise ValueError( + f"{prefix} is {code_dtype}/{scale_dtype}, not an F8_E4M3 resident" + ) + return codes.reshape(shape), scales.reshape(scale_shape) + + +def check_exactness(codes: np.ndarray, scales: np.ndarray) -> dict[str, Any]: + """Compare the mxfp8 byte copy against today's bf16 decode, bit for bit.""" + + packed, group_scales = repack_fp8_block_as_mxfp8(codes, scales, block_size=FP8_BLOCK) + reference = mx.array(dequantize_fp8_block(codes, scales, block_size=FP8_BLOCK)).astype( + mx.bfloat16 + ) + restored = mx.dequantize( + mx.array(packed), + mx.array(group_scales), + group_size=32, + bits=8, + mode="mxfp8", + dtype=mx.bfloat16, + ) + mismatches = int(mx.sum(restored.view(mx.uint16) != reference.view(mx.uint16)).item()) + return { + "elements": int(codes.size), + "bit_mismatches": mismatches, + "bf16_bytes": int(codes.size) * 2, + "mxfp8_bytes": int(packed.nbytes + group_scales.nbytes), + } + + +def _median_call_ms( + fn: Callable[[int], mx.array], *, iterations: int, warmup: int +) -> float: + for i in range(warmup): + mx.eval(fn(i)) + started = time.perf_counter() + for i in range(iterations): + mx.eval(fn(i)) + return (time.perf_counter() - started) * 1_000.0 / iterations + + +def time_variants( + codes: np.ndarray, + scales: np.ndarray, + *, + copies: int, + iterations: int, + warmup: int, + rounds: int, + seed: int, + groups: int = 1, +) -> dict[str, Any]: + """Per-call M=1 latency for bf16, mxfp8 and affine-8 g64, interleaved. + + ``groups > 1`` times the grouped form: rows split into ``groups`` equal + blocks, each multiplied by its own activation row, as ``wo_a`` runs. + """ + + rows, cols = codes.shape + if groups < 1 or rows % groups: + raise ValueError(f"{rows} rows do not split into {groups} groups") + packed, group_scales = repack_fp8_block_as_mxfp8(codes, scales, block_size=FP8_BLOCK) + dense = mx.array(dequantize_fp8_block(codes, scales, block_size=FP8_BLOCK)).astype( + mx.bfloat16 + ) + if groups > 1: + packed = packed.reshape(groups, rows // groups, -1) + group_scales = group_scales.reshape(groups, rows // groups, -1) + dense = dense.reshape(groups, rows // groups, cols) + affine_w, affine_s, affine_b = mx.quantize(dense, group_size=64, bits=8) + # Distinct buffers per copy so consecutive calls cannot hit a cache. + bf16_copies = [dense + mx.zeros_like(dense) for _ in range(copies)] + packed_copies = [mx.array(packed) for _ in range(copies)] + scale_copies = [mx.array(group_scales) for _ in range(copies)] + affine_copies = [ + (affine_w + mx.zeros_like(affine_w), affine_s + 0, affine_b + 0) for _ in range(copies) + ] + mx.eval(bf16_copies, packed_copies, scale_copies, affine_copies) + x_shape = (1, groups, 1, cols) if groups > 1 else (1, cols) + x = mx.random.normal(x_shape, key=mx.random.key(seed)).astype(mx.bfloat16) + + def bf16(i: int) -> mx.array: + return x @ bf16_copies[i % copies].swapaxes(-1, -2) + + def mxfp8(i: int) -> mx.array: + j = i % copies + return mx.quantized_matmul( + x, packed_copies[j], scale_copies[j], transpose=True, group_size=32, bits=8, mode="mxfp8" + ) + + def affine8(i: int) -> mx.array: + w, s, b = affine_copies[i % copies] + return mx.quantized_matmul(x, w, s, b, transpose=True, group_size=64, bits=8) + + variants = {"bf16": bf16, "mxfp8": mxfp8, "affine8_g64": affine8} + samples: dict[str, list[float]] = {name: [] for name in variants} + order = list(variants) + for round_index in range(rounds): + # Rotate the order so no variant always runs first or last. + rotated = order[round_index % len(order) :] + order[: round_index % len(order)] + for name in rotated: + samples[name].append( + _median_call_ms(variants[name], iterations=iterations, warmup=warmup) + ) + weight_bytes = { + "bf16": rows * cols * 2, + "mxfp8": int(packed.nbytes + group_scales.nbytes), + "affine8_g64": int(affine_w.nbytes + affine_s.nbytes + affine_b.nbytes), + } + out: dict[str, Any] = {} + for name, values in samples.items(): + median_ms = statistics.median(values) + out[name] = { + "median_ms_per_call": median_ms, + "round_ms": values, + "weight_bytes": weight_bytes[name], + "effective_gb_per_s": weight_bytes[name] / (median_ms / 1_000.0) / 1e9, + } + del bf16_copies, packed_copies, scale_copies, affine_copies + return out + + +def verdict( + exactness: Mapping[str, Mapping[str, Any]], + timings: Mapping[str, Mapping[str, Mapping[str, Any]]] | None, + *, + speed_valid: bool, +) -> dict[str, Any]: + """Apply the kill criteria. Exactness alone can kill; speed needs a GPU.""" + + mismatches = sum(int(row["bit_mismatches"]) for row in exactness.values()) + result: dict[str, Any] = { + "bit_mismatches": mismatches, + "exact": mismatches == 0, + "speed_valid": bool(speed_valid and timings), + } + reasons: list[str] = [] + if mismatches: + reasons.append(f"{mismatches} dequantized weights differ from the bf16 decode") + if timings: + totals = { + variant: sum(float(rows[variant]["median_ms_per_call"]) for rows in timings.values()) + for variant in ("bf16", "mxfp8", "affine8_g64") + } + speedup = totals["bf16"] / totals["mxfp8"] + vs_affine = totals["mxfp8"] / totals["affine8_g64"] + result.update( + { + "summed_median_ms": totals, + "mxfp8_speedup_over_bf16": speedup, + "mxfp8_time_over_affine8": vs_affine, + } + ) + if speed_valid: + if speedup < MIN_SPEEDUP_OVER_BF16: + reasons.append( + f"mxfp8 is {speedup:.3f}x bf16, below the {MIN_SPEEDUP_OVER_BF16}x bar" + ) + if vs_affine > MAX_SLOWDOWN_VS_AFFINE8: + reasons.append( + f"mxfp8 takes {vs_affine:.3f}x affine-8's time, above {MAX_SLOWDOWN_VS_AFFINE8}x" + ) + if reasons: + result["verdict"] = "KILL" + elif result["speed_valid"]: + result["verdict"] = "PASS" + else: + result["verdict"] = "EXACT_ONLY" # exactness passed; rerun on an Apple GPU for speed + result["reasons"] = reasons + return result + + +def run( + *, + tensors: Sequence[str], + layer: int, + checkpoint: Path | None, + synthetic_shapes: Mapping[str, tuple[int, int]] | None, + copies: int, + iterations: int, + warmup: int, + rounds: int, + seed: int, + exactness_only: bool, +) -> dict[str, Any]: + if (checkpoint is None) == (synthetic_shapes is None): + raise ValueError("pass exactly one of checkpoint or synthetic_shapes") + rng = np.random.default_rng(seed) + reader = _CheckpointReader(checkpoint) if checkpoint is not None else None + on_gpu = mx.default_device() == mx.gpu + exactness: dict[str, dict[str, Any]] = {} + timings: dict[str, dict[str, Any]] = {} + shapes: dict[str, list[int]] = {} + for tensor in tensors: + if reader is not None: + codes, scales = reader.fp8_pair(f"layers.{layer}.{tensor}") + else: + codes, scales = synthetic_fp8_pair(rng, synthetic_shapes[tensor]) + shapes[tensor] = list(codes.shape) + exactness[tensor] = check_exactness(codes, scales) + if not exactness_only: + timings[tensor] = time_variants( + codes, + scales, + copies=copies, + iterations=iterations, + warmup=warmup, + rounds=rounds, + seed=seed, + groups=GROUPED_TENSORS.get(tensor, 1), + ) + mx.clear_cache() + return { + "record_type": RECORD_TYPE, + "created_utc": _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds"), + "source": "synthetic" if reader is None else str(checkpoint), + "layer": None if reader is None else layer, + "shapes": shapes, + "device": str(mx.default_device()), + "device_info": {k: (v if isinstance(v, (int, float, str)) else str(v)) for k, v in mx.device_info().items()} + if on_gpu + else {}, + "mlx_version": mx.__version__, + "platform": platform.platform(), + "settings": { + "copies": copies, + "iterations": iterations, + "warmup": warmup, + "rounds": rounds, + "seed": seed, + }, + "exactness": exactness, + "timings": timings, + "verdict": verdict(exactness, timings or None, speed_valid=on_gpu), + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--checkpoint", type=Path, help="resident package or source checkpoint dir") + source.add_argument("--synthetic", action="store_true", help="random FP8 at V4-Flash shapes") + parser.add_argument("--layer", type=int, default=20) + parser.add_argument("--tensors", default=",".join(V4_FLASH_RESIDENT_SHAPES)) + parser.add_argument("--copies", type=int, default=8) + parser.add_argument("--iterations", type=int, default=200) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--rounds", type=int, default=5) + parser.add_argument("--seed", type=int, default=20261105) + parser.add_argument("--exactness-only", action="store_true") + parser.add_argument("--no-lock", action="store_true", help="skip the heavy-job lock") + parser.add_argument("--output-json", type=Path) + args = parser.parse_args(argv) + + forbidden = [name for name in _FORBIDDEN_ENV if name in os.environ] + if forbidden: + parser.error("unset custom MLX wired limits first: " + ", ".join(forbidden)) + tensors = [name.strip() for name in args.tensors.split(",") if name.strip()] + unknown = [name for name in tensors if args.synthetic and name not in V4_FLASH_RESIDENT_SHAPES] + if unknown: + parser.error("no synthetic shape for: " + ", ".join(unknown)) + + def _go() -> dict[str, Any]: + return run( + tensors=tensors, + layer=args.layer, + checkpoint=args.checkpoint, + synthetic_shapes=V4_FLASH_RESIDENT_SHAPES if args.synthetic else None, + copies=args.copies, + iterations=args.iterations, + warmup=args.warmup, + rounds=args.rounds, + seed=args.seed, + exactness_only=args.exactness_only, + ) + + if args.no_lock: + result = _go() + else: + from keep.quality.dsv4_teacher_agreement import heavy_job_lock + + with heavy_job_lock(HEAVY_JOB_LOCK, holder="bench_dsv4_mxfp8_residents"): + result = _go() + text = json.dumps(result, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(text + "\n") + print(text) + return 0 if result["verdict"]["verdict"] != "KILL" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/keep/convert/fp8_block.py b/src/keep/convert/fp8_block.py index a0f0b83..031945f 100644 --- a/src/keep/convert/fp8_block.py +++ b/src/keep/convert/fp8_block.py @@ -19,13 +19,27 @@ Pure numpy so it runs headless; the MLX fast path can come later if decode throughput matters. + +:func:`repack_fp8_block_as_mxfp8` is the lossless alternative to decoding: it +re-lays the same e4m3 bytes and ue8m0 exponents out in MLX's ``mxfp8`` layout +(group 32 along the input axis), so a resident can stay at 8.25 bits per weight +instead of expanding to bf16, with every dequantized value bit-identical. """ from __future__ import annotations import numpy as np -__all__ = ["decode_e4m3", "decode_ue8m0", "dequantize_fp8_block"] +__all__ = [ + "MXFP8_GROUP_SIZE", + "decode_e4m3", + "decode_ue8m0", + "dequantize_fp8_block", + "repack_fp8_block_as_mxfp8", +] + +#: MLX ``mxfp8`` supports exactly one group size: 32 elements per E8M0 scale. +MXFP8_GROUP_SIZE = 32 _E4M3_TABLE: np.ndarray | None = None @@ -130,3 +144,112 @@ def dequantize_fp8_block( c0, c1 = j * block_cols, min((j + 1) * block_cols, cols) decoded[r0:r1, c0:c1] *= row_scales[j] return decoded + + +def _power_of_two_scales_as_ue8m0(scales: np.ndarray) -> np.ndarray: + """Encode literal power-of-two multipliers as ue8m0 bytes, or refuse. + + Only exact powers of two in ``2**-127 .. 2**127`` have an E8M0 encoding. + Anything else would need rounding, and rounding is exactly what a lossless + repack must never do, so it raises instead. + """ + + values = scales.astype(np.float64) + if not np.all(np.isfinite(values)) or np.any(values <= 0.0): + raise ValueError("power-of-two block scales must be finite and positive") + mantissa, exponent = np.frexp(values) + if np.any(mantissa != 0.5): + raise ValueError( + "block scales are not all exact powers of two; an mxfp8 repack " + "would have to round them, so it is refused" + ) + unbiased = exponent.astype(np.int64) - 1 + if np.any(unbiased < -127) or np.any(unbiased > 127): + raise ValueError("power-of-two block scales are outside the E8M0 range") + return (unbiased + 127).astype(np.uint8) + + +def repack_fp8_block_as_mxfp8( + codes: np.ndarray, + scales: np.ndarray, + *, + block_size: tuple[int, int] = (128, 128), + group_size: int = MXFP8_GROUP_SIZE, +) -> tuple[np.ndarray, np.ndarray]: + """Re-lay block-scaled e4m3 weights out as MLX ``mxfp8`` without changing a bit. + + Returns ``(packed, group_scales)``: + + - ``packed``: ``uint32`` of shape ``(rows, cols // 4)``, the unchanged e4m3 + code bytes read little-endian four to a word, which is the element order + ``mx.dequantize(..., mode="mxfp8")`` expects. + - ``group_scales``: ``uint8`` of shape ``(rows, cols // 32)``, each block's + ue8m0 exponent repeated over the rows and the 32-wide column groups the + block covers. + + Dequantizing the result reproduces :func:`dequantize_fp8_block` exactly: + e4m3 times a power of two is representable in float32 and bf16 alike. + Building the same weights with ``mx.quantize(..., mode="mxfp8")`` instead is + *not* exact -- it re-derives each group's exponent by rounding -- so this + function is the only supported way to produce an mxfp8 resident. + + Requires ``cols % 32 == 0`` and ``block_size[1] % 32 == 0`` so every + 32-wide group sits inside one source block. ue8m0 byte 255 (NaN) is + refused rather than bound. + """ + + if not isinstance(codes, np.ndarray): + raise ValueError(f"codes must be a numpy ndarray, got {type(codes).__name__}") + if not isinstance(scales, np.ndarray): + raise ValueError(f"scales must be a numpy ndarray, got {type(scales).__name__}") + if codes.dtype != np.uint8: + raise ValueError(f"e4m3 codes must be uint8, got {codes.dtype}") + if codes.ndim != 2: + raise ValueError(f"codes must be 2-D, got shape {codes.shape}") + if group_size != MXFP8_GROUP_SIZE: + raise ValueError( + f"mxfp8 has a fixed group size of {MXFP8_GROUP_SIZE}, got {group_size!r}" + ) + block_rows, block_cols = block_size + for label, value in (("block_size[0]", block_rows), ("block_size[1]", block_cols)): + if ( + isinstance(value, bool) + or not isinstance(value, (int, np.integer)) + or value <= 0 + ): + raise ValueError(f"{label} must be a positive int, got {value!r}") + rows, cols = codes.shape + if cols % group_size != 0: + raise ValueError( + f"input dimension {cols} is not a multiple of the mxfp8 group size {group_size}" + ) + if block_cols % group_size != 0: + raise ValueError( + f"block width {block_cols} is not a multiple of the mxfp8 group size " + f"{group_size}, so a group would straddle two source scales" + ) + expected = (-(-rows // block_rows), -(-cols // block_cols)) + if scales.shape != expected: + raise ValueError( + f"scales shape {scales.shape} does not match " + f"{expected} blocks for weight {codes.shape} " + f"at block size {block_size}" + ) + if scales.dtype == np.uint8: + if np.any(scales == 255): + raise ValueError("ue8m0 byte 255 is NaN; refusing to repack a NaN block scale") + exponents = scales + elif np.issubdtype(scales.dtype, np.floating): + exponents = _power_of_two_scales_as_ue8m0(scales) + else: + raise ValueError( + f"scales must be uint8 (ue8m0-encoded) or a floating dtype holding " + f"powers of two, got dtype {scales.dtype}" + ) + + # ``astype`` copies, so the result never aliases the caller's code buffer. + packed = np.ascontiguousarray(codes).view(" int: @@ -2822,6 +2833,9 @@ def to_dict(self) -> dict[str, object]: "skipped_out_of_scope_tensors": list(self.skipped_out_of_scope_tensors), "skipped_unmatched_tensors": list(self.skipped_unmatched_tensors), "missing_model_parameters": list(self.missing_model_parameters), + "resident_format": self.resident_format, + "mxfp8_repacked_tensors": list(self.mxfp8_repacked_tensors), + "mxfp8_scale_parameters": list(self.mxfp8_scale_parameters), } @@ -2985,6 +2999,113 @@ def _decode_source_tensor( return mx.array(values), "direct" +RESIDENT_FORMATS = ("bf16", "mxfp8") + + +def _repack_source_tensor_mxfp8( + dtype: str, + raw: bytes, + shape: tuple[int, ...], + scale: tuple[str, bytes, tuple[int, ...]] | None, +) -> tuple[mx.array, mx.array]: + """Byte-copy an FP8 resident into MLX mxfp8 ``(weight, scales)`` arrays. + + Same dtype guards as the decode path. The values this binds dequantize to + exactly what :func:`_decode_source_tensor` would have bound in bf16. + """ + + if scale is None or dtype != _FP8_CODE_DTYPE: + raise ValueError( + f"mxfp8 repack needs {_FP8_CODE_DTYPE} codes with a .scale companion, " + f"got {dtype} {'with' if scale is not None else 'without'} a scale" + ) + scale_dtype, scale_raw, scale_shape = scale + if scale_dtype != _FP8_SCALE_DTYPE: + raise ValueError(f"block scales must be {_FP8_SCALE_DTYPE}, got {scale_dtype}") + codes = np.frombuffer(raw, dtype=np.uint8).reshape(shape) + scales = np.frombuffer(scale_raw, dtype=np.uint8).reshape(scale_shape) + packed, group_scales = repack_fp8_block_as_mxfp8(codes, scales, block_size=_FP8_BLOCK) + return mx.array(packed), mx.array(group_scales) + + +def _mxfp8_eligible_module_paths( + model: nn.Module, + plan: Sequence[tuple[str, str, str | None]], +) -> dict[str, str]: + """Map ``target .weight`` -> module path for residents that can stay mxfp8. + + Eligible means: an FP8 source (it has a ``.scale``), a plain ``nn.Linear`` + target or the ``wo_a`` ``MultiLinear`` (whose flat source rows regroup by + a pure reshape), and an input width divisible by the mxfp8 group. Anything + else stays on the bf16 decode path. + """ + + modules = dict(model.named_modules()) + eligible: dict[str, str] = {} + if _FP8_BLOCK[1] % MXFP8_GROUP_SIZE != 0: + return eligible + for source, target, scale_name in plan: + if scale_name is None or not target.endswith(".weight"): + continue + module_path = target[: -len(".weight")] + module = modules.get(module_path) + if type(module) is nn.Linear: + if module.weight.ndim != 2: + continue + elif type(module) is MultiLinear and _RESHAPE_ALLOWED_RE.search(source): + if module.weight.ndim != 3: + continue + else: + continue + if module.weight.shape[-1] % MXFP8_GROUP_SIZE != 0: + continue + eligible[target] = module_path + return eligible + + +def _swap_linears_for_mxfp8(model: nn.Module, module_paths: Iterable[str]) -> None: + """Replace the named linear modules with weights-only mxfp8 ones. + + ``nn.Linear`` becomes ``nn.QuantizedLinear`` and ``MultiLinear`` becomes + mlx-lm's ``QuantizedMultiLinear``. ``quantize_input=False`` is explicit: + activations stay in floating point, so the bound weights reproduce the + bf16 teacher's weights exactly. The placeholder quantization of the + skeleton weights is lazy and is overwritten by the bind before anything + evaluates it. + """ + + wanted = set(module_paths) + if not wanted: + return + nn.quantize( + model, + group_size=MXFP8_GROUP_SIZE, + bits=8, + mode="mxfp8", + quantize_input=False, + class_predicate=lambda path, module: path in wanted, + ) + modules = dict(model.named_modules()) + not_swapped = sorted( + path + for path in wanted + if type(modules.get(path)) not in (nn.QuantizedLinear, QuantizedMultiLinear) + or getattr(modules.get(path), "mode", None) != "mxfp8" + ) + if not_swapped: + raise RuntimeError( + "mxfp8 module swap left plain modules behind: " + ", ".join(not_swapped[:10]) + ) + + +def _linear_input_dims(module: nn.Module) -> int: + """Input width of a plain or quantized linear layer.""" + + if isinstance(module, nn.QuantizedLinear): + return int(module.weight.shape[1]) * 32 // int(module.bits) + return int(module.weight.shape[1]) + + def bind_deepseek_v4_flash_non_vq_weights( model: DeepseekV4FlashVQModel, checkpoint_dir: str | Path, @@ -2993,6 +3114,7 @@ def bind_deepseek_v4_flash_non_vq_weights( include_mtp: bool = True, strict: bool = True, compute_dtype: mx.Dtype = mx.bfloat16, + resident_format: str = "bf16", ) -> DeepseekV4FlashNonVQBindReport: """Bind every non-routed tensor, shard by shard, off the safetensors index. @@ -3001,12 +3123,25 @@ def bind_deepseek_v4_flash_non_vq_weights( and ``F32`` tensors load directly; ``I64`` routing tables keep their native dtype. Routed experts are skipped -- they are the VQ path. + ``resident_format="mxfp8"`` (opt-in) keeps every FP8 resident that lands on + a plain ``nn.Linear`` or on ``wo_a``'s ``MultiLinear`` as its source bytes + instead: the module becomes a weights-only ``nn.QuantizedLinear`` (or + mlx-lm ``QuantizedMultiLinear``) in ``mode="mxfp8"``, holding the unchanged + e4m3 codes and their ue8m0 exponents (8.25 bits per weight instead of 16), + and dequantizes bit-identically to the bf16 decode. Non-FP8 tensors and + inputs not divisible by 32 keep the default path. The default, ``"bf16"``, + is unchanged. + ``layers`` restricts the backbone layers touched (a layer-0 smoke test does not need 163 GB of I/O). ``strict`` is only meaningful for a full bind: it is forced off when ``layers`` or ``include_mtp=False`` deliberately leaves parameters unbound. """ + if resident_format not in RESIDENT_FORMATS: + raise ValueError( + f"resident_format must be one of {RESIDENT_FORMATS}, got {resident_format!r}" + ) checkpoint_path = Path(checkpoint_dir) weight_map = _load_weight_map(checkpoint_path) partial_scope = layers is not None or not include_mtp @@ -3078,6 +3213,14 @@ def bind_deepseek_v4_flash_non_vq_weights( + ", ".join(unmatched[:20]) ) + mxfp8_targets: dict[str, str] = {} + if resident_format == "mxfp8": + mxfp8_targets = _mxfp8_eligible_module_paths(model, plan) + _swap_linears_for_mxfp8(model, mxfp8_targets.values()) + target_params = {key: value for key, value in tree_flatten(model.parameters())} + mxfp8_repacked: list[str] = [] + mxfp8_scales: list[str] = [] + # Group by shard so each file is opened (and its header parsed) once. by_shard: dict[str, list[tuple[str, str, str | None]]] = {} for source, target, scale_name in plan: @@ -3116,6 +3259,43 @@ def bind_deepseek_v4_flash_non_vq_weights( scale_tensor.shape, ) + if target in mxfp8_targets: + packed, group_scales = _repack_source_tensor_mxfp8( + tensor.dtype, raw, tensor.shape, scale_payload + ) + scales_target = f"{mxfp8_targets[target]}.scales" + regrouped = False + for name, value in ((target, packed), (scales_target, group_scales)): + expected = target_params.get(name) + if ( + expected is not None + and tuple(value.shape) != tuple(expected.shape) + and _RESHAPE_ALLOWED_RE.search(source) + and value.size == expected.size + ): + # ``wo_a``: flat source rows regroup into + # MultiLinear's (groups, rows, cols) layout. Rows + # carry their own expanded scales, so this is a + # pure relabel of whole rows. + value = value.reshape(expected.shape) + regrouped = True + if expected is None or tuple(value.shape) != tuple(expected.shape): + raise ValueError( + f"{source} repacks to {name} shape {tuple(value.shape)} " + f"but the mxfp8 module expects " + f"{None if expected is None else tuple(expected.shape)}" + ) + batch.append((name, value)) + if regrouped: + reshaped.append(source) + mxfp8_repacked.append(source) + mxfp8_scales.append(scales_target) + if target.startswith("mtp_drafter."): + bound_mtp.append(target) + else: + bound_backbone.append(target) + continue + value, kind = _decode_source_tensor( tensor.dtype, raw, @@ -3167,7 +3347,7 @@ def bind_deepseek_v4_flash_non_vq_weights( # config -- for a bind that touched one layer. mx.eval([value for _, value in batch]) - bound_all = set(bound_backbone) | set(bound_mtp) + bound_all = set(bound_backbone) | set(bound_mtp) | set(mxfp8_scales) missing = tuple(sorted(set(target_params) - bound_all)) report = DeepseekV4FlashNonVQBindReport( bound_backbone_parameters=tuple(sorted(bound_backbone)), @@ -3182,6 +3362,9 @@ def bind_deepseek_v4_flash_non_vq_weights( skipped_out_of_scope_tensors=tuple(sorted(skipped_scope)), skipped_unmatched_tensors=tuple(sorted(unmatched)), missing_model_parameters=missing, + resident_format=resident_format, + mxfp8_repacked_tensors=tuple(sorted(mxfp8_repacked)), + mxfp8_scale_parameters=tuple(sorted(mxfp8_scales)), ) if strict and missing: raise ValueError( diff --git a/src/ramp/models/dsv4_composite_loader.py b/src/ramp/models/dsv4_composite_loader.py index e0fb054..9ddc0d4 100644 --- a/src/ramp/models/dsv4_composite_loader.py +++ b/src/ramp/models/dsv4_composite_loader.py @@ -636,13 +636,25 @@ def validate_dsv4_composite_inputs( def load_authenticated_dsv4_composite( inputs: Dsv4ValidatedCompositeInputs, + *, + resident_format: str = "bf16", ) -> Dsv4LoadedComposite: - """Construct and bind exactly the authenticated resident+routed model.""" + """Construct and bind exactly the authenticated resident+routed model. + + ``resident_format="mxfp8"`` keeps FP8 residents as their source bytes + (weights-only mxfp8, bit-identical weights) instead of decoding them to + bf16; see :func:`bind_deepseek_v4_flash_non_vq_weights`. The artifact and + its identity are the same either way -- only the in-memory layout differs. + """ args = deepseek_v4_flash_args_from_config(inputs.config) model = DeepseekV4FlashVQModel(args, with_mtp=True) non_vq = bind_deepseek_v4_flash_non_vq_weights( - model, inputs.resident_dir, include_mtp=True, strict=True + model, + inputs.resident_dir, + include_mtp=True, + strict=True, + resident_format=resident_format, ) backbone = bind_deepseek_v4_flash_vq_experts( model, diff --git a/tests/test_bench_dsv4_mxfp8_residents.py b/tests/test_bench_dsv4_mxfp8_residents.py new file mode 100644 index 0000000..ec8eb40 --- /dev/null +++ b/tests/test_bench_dsv4_mxfp8_residents.py @@ -0,0 +1,168 @@ +"""The mxfp8-resident kill test: its exactness half and its verdict rules.""" + +from __future__ import annotations + +import json +import struct +from pathlib import Path + +import numpy as np +import pytest + +mx = pytest.importorskip("mlx.core") + +from benchmarks import bench_dsv4_mxfp8_residents as bench # noqa: E402 + + +def _row(bf16: float, mxfp8: float, affine8: float) -> dict: + return { + "bf16": {"median_ms_per_call": bf16}, + "mxfp8": {"median_ms_per_call": mxfp8}, + "affine8_g64": {"median_ms_per_call": affine8}, + } + + +_EXACT = {"t": {"bit_mismatches": 0}} + + +def test_verdict_passes_only_when_exact_and_fast_on_a_gpu(): + timings = {"t": _row(bf16=2.0, mxfp8=1.0, affine8=1.0)} + assert bench.verdict(_EXACT, timings, speed_valid=True)["verdict"] == "PASS" + # Same numbers from a CPU run prove exactness only. + assert bench.verdict(_EXACT, timings, speed_valid=False)["verdict"] == "EXACT_ONLY" + assert bench.verdict(_EXACT, None, speed_valid=True)["verdict"] == "EXACT_ONLY" + + +@pytest.mark.parametrize( + ("bf16", "mxfp8", "affine8", "needle"), + [ + (1.4, 1.0, 1.0, "below the 1.5x bar"), + (2.0, 1.2, 1.0, "affine-8"), + ], +) +def test_verdict_kills_on_either_speed_criterion(bf16, mxfp8, affine8, needle): + result = bench.verdict(_EXACT, {"t": _row(bf16, mxfp8, affine8)}, speed_valid=True) + assert result["verdict"] == "KILL" + assert any(needle in reason for reason in result["reasons"]) + + +def test_verdict_kills_on_any_bit_mismatch_even_without_timings(): + result = bench.verdict({"t": {"bit_mismatches": 1}}, None, speed_valid=False) + assert result["verdict"] == "KILL" and result["exact"] is False + + +def test_speed_is_judged_on_the_summed_decode_step_not_per_tensor(): + timings = { + "fast": _row(bf16=3.0, mxfp8=1.0, affine8=1.0), + "slow": _row(bf16=1.0, mxfp8=1.0, affine8=1.0), + } + result = bench.verdict(_EXACT, timings, speed_valid=True) + assert result["mxfp8_speedup_over_bf16"] == pytest.approx(2.0) + assert result["verdict"] == "PASS" + + +def test_synthetic_run_is_exact_and_reports_every_variant(): + result = bench.run( + tensors=["a", "b"], + layer=0, + checkpoint=None, + synthetic_shapes={"a": (64, 256), "b": (130, 192)}, # ragged row and column blocks + copies=2, + iterations=2, + warmup=1, + rounds=2, + seed=0, + exactness_only=False, + ) + assert result["record_type"] == bench.RECORD_TYPE + assert result["verdict"]["exact"] is True + for tensor in ("a", "b"): + assert set(result["timings"][tensor]) == {"bf16", "mxfp8", "affine8_g64"} + assert result["exactness"][tensor]["mxfp8_bytes"] < result["exactness"][tensor]["bf16_bytes"] + if mx.default_device() != mx.gpu: + assert result["verdict"]["verdict"] == "EXACT_ONLY" + + +def _write_fp8_checkpoint(root: Path, tensors: dict[str, tuple[np.ndarray, np.ndarray]]) -> None: + header: dict[str, object] = {} + blobs: list[bytes] = [] + offset = 0 + for prefix, (codes, scales) in sorted(tensors.items()): + for suffix, dtype, array in ( + ("weight", "F8_E4M3", codes), + ("scale", "F8_E8M0", scales), + ): + raw = array.tobytes() + header[f"{prefix}.{suffix}"] = { + "dtype": dtype, + "shape": list(array.shape), + "data_offsets": [offset, offset + len(raw)], + } + offset += len(raw) + blobs.append(raw) + raw_header = json.dumps(header, separators=(",", ":")).encode() + raw_header += b" " * ((-len(raw_header)) % 8) + shard = "model-00001-of-00001.safetensors" + with (root / shard).open("wb") as handle: + handle.write(struct.pack(" 0 + + +# --------------------------------------------------------------------------- +# Opt-in mxfp8 residents: the FP8 bytes stay as they shipped +# --------------------------------------------------------------------------- + + +def _named(model) -> dict: + return dict(model.named_modules()) + + +def test_mxfp8_resident_bind_keeps_the_source_bytes_bit_identically(tmp_path): + from ramp.models.deepseek_v4_flash_adapter import _linear_input_dims + + args = _mtp_args() + _write_synthetic_checkpoint(tmp_path, args, with_mtp=True) + reference = DeepseekV4FlashVQModel(args) + ref_report = bind_deepseek_v4_flash_non_vq_weights(reference, tmp_path, strict=True) + model = DeepseekV4FlashVQModel(args) + report = bind_deepseek_v4_flash_non_vq_weights( + model, tmp_path, strict=True, resident_format="mxfp8" + ) + + assert ref_report.resident_format == "bf16" and ref_report.mxfp8_repacked_tensors == () + assert report.resident_format == "mxfp8" + assert report.missing_model_parameters == () + # Logical accounting is unchanged: the .scales are bookkeeping, not tensors. + assert report.bound_count == ref_report.bound_count + assert set(report.bound_backbone_parameters) == set(ref_report.bound_backbone_parameters) + assert len(report.mxfp8_scale_parameters) == len(report.mxfp8_repacked_tensors) + + repacked = set(report.mxfp8_repacked_tensors) + # Plain linears with a 32-divisible input width stay FP8 ... + for name in ( + "layers.0.attn.wq_a.weight", + "layers.0.attn.wkv.weight", + "layers.0.ffn.shared_experts.w1.weight", + "layers.0.ffn.shared_experts.w2.weight", + "layers.0.ffn.shared_experts.w3.weight", + "mtp.0.main_proj.weight", + ): + assert name in repacked, name + # ... while inputs too narrow for a 32-wide group keep the bf16 decode + # (wo_a's per-group input is 8 wide in this fixture; see the next test). + assert "layers.0.attn.wo_a.weight" not in repacked + assert "layers.0.attn.wo_a.weight" in report.fp8_block_decoded_tensors + assert "layers.0.attn.wq_b.weight" not in repacked # input width 8 + assert set(report.fp8_block_decoded_tensors).isdisjoint(repacked) + + ref_modules, modules = _named(reference), _named(model) + for scales_name in report.mxfp8_scale_parameters: + path = scales_name[: -len(".scales")] + quantized, plain = modules[path], ref_modules[path] + assert type(quantized) is nn.QuantizedLinear, path + assert (quantized.mode, quantized.group_size, quantized.bits) == ("mxfp8", 32, 8) + assert quantized.weight.dtype == mx.uint32 and quantized.scales.dtype == mx.uint8 + restored = mx.dequantize( + quantized.weight, + quantized.scales, + group_size=32, + bits=8, + mode="mxfp8", + dtype=mx.bfloat16, + ) + assert restored.shape == plain.weight.shape + assert np.array_equal( + np.array(restored.view(mx.uint16)), np.array(plain.weight.view(mx.uint16)) + ), path + assert _linear_input_dims(quantized) == _linear_input_dims(plain) + + x = mx.array(np.random.default_rng(5).normal(size=(2, plain.weight.shape[1])), dtype=mx.float32) + got, want = np.array(quantized(x)), np.array(x @ plain.weight.astype(mx.float32).T) + bound = 1e-5 * (np.abs(np.array(x)) @ np.abs(np.array(plain.weight.astype(mx.float32))).T) + assert np.all(np.abs(got - want) <= bound + 1e-6), path + + # Everything not repacked is bound exactly as the default path binds it. + ref_params = dict(tree_flatten(reference.parameters())) + params = dict(tree_flatten(model.parameters())) + repacked_modules = {name[: -len(".scales")] for name in report.mxfp8_scale_parameters} + def as_numpy(array): + # numpy has no bfloat16; widening to float32 is exact. + if mx.issubdtype(array.dtype, mx.floating): + array = array.astype(mx.float32) + return np.array(array) + + for name, value in ref_params.items(): + if name.rsplit(".", 1)[0] in repacked_modules: + continue + assert np.array_equal(as_numpy(params[name]), as_numpy(value), equal_nan=True), name + + +def test_mxfp8_is_opt_in_and_the_format_is_validated(tmp_path): + args = _tiny_args(num_hidden_layers=1, num_hash_layers=0, compress_ratios=[0]) + _write_synthetic_checkpoint(tmp_path, args) + model = DeepseekV4FlashVQModel(args, with_mtp=False) + with pytest.raises(ValueError, match="resident_format"): + bind_deepseek_v4_flash_non_vq_weights( + model, tmp_path, include_mtp=False, resident_format="fp8" + ) + report = bind_deepseek_v4_flash_non_vq_weights(model, tmp_path, include_mtp=False) + assert report.resident_format == "bf16" + assert not any( + isinstance(module, nn.QuantizedLinear) for _, module in model.named_modules() + ) + + +def test_mxfp8_bind_regroups_wo_a_into_a_quantized_multilinear(tmp_path): + from mlx_lm.models.mla import QuantizedMultiLinear + + # 8 heads x 8 dims over 2 groups gives wo_a a 32-wide per-group input. + args = _tiny_args( + num_hidden_layers=1, num_hash_layers=0, compress_ratios=[0], num_attention_heads=8 + ) + _write_synthetic_checkpoint(tmp_path, args) + reference = DeepseekV4FlashVQModel(args, with_mtp=False) + bind_deepseek_v4_flash_non_vq_weights(reference, tmp_path, include_mtp=False) + model = DeepseekV4FlashVQModel(args, with_mtp=False) + report = bind_deepseek_v4_flash_non_vq_weights( + model, tmp_path, include_mtp=False, strict=True, resident_format="mxfp8" + ) + + assert "layers.0.attn.wo_a.weight" in report.mxfp8_repacked_tensors + assert "layers.0.attn.wo_a.weight" in report.reshaped_tensors + assert report.missing_model_parameters == () + wo_a, plain = model.model.layers[0].attn.wo_a, reference.model.layers[0].attn.wo_a + assert type(wo_a) is QuantizedMultiLinear and wo_a.mode == "mxfp8" + restored = mx.dequantize( + wo_a.weight, wo_a.scales, group_size=32, bits=8, mode="mxfp8", dtype=mx.bfloat16 + ) + assert restored.shape == plain.weight.shape + assert np.array_equal( + np.array(restored.view(mx.uint16)), np.array(plain.weight.view(mx.uint16)) + ) + groups, _, width = plain.weight.shape + x = mx.array(np.random.default_rng(6).normal(size=(1, groups, 3, width)), dtype=mx.float32) + dense = plain.weight.astype(mx.float32) + got, want = np.array(wo_a(x)), np.array(x @ dense.swapaxes(-1, -2)) + bound = 1e-5 * (np.abs(np.array(x)) @ np.abs(np.array(dense)).swapaxes(-1, -2)) + assert np.all(np.abs(got - want) <= bound + 1e-6) diff --git a/tests/test_fp8_block_dequant.py b/tests/test_fp8_block_dequant.py index f765f61..33f8b32 100644 --- a/tests/test_fp8_block_dequant.py +++ b/tests/test_fp8_block_dequant.py @@ -9,6 +9,7 @@ decode_e4m3, decode_ue8m0, dequantize_fp8_block, + repack_fp8_block_as_mxfp8, ) @@ -170,3 +171,144 @@ def test_block_dequant_rejects_non_ndarray_inputs(): dequantize_fp8_block([[0x38]], scales) # type: ignore[arg-type] with pytest.raises(ValueError): dequantize_fp8_block(codes, [[1.0]]) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# Lossless mxfp8 repack (residents kept as their source bytes) +# -------------------------------------------------------------------------- + + +def _finite_fp8_pair(rng, rows, cols, block=(128, 128)): + codes = rng.integers(0, 256, size=(rows, cols)).astype(np.uint8) + codes = np.where((codes & 0x7F) == 0x7F, 0x7E, codes).astype(np.uint8) + blocks = (-(-rows // block[0]), -(-cols // block[1])) + scales = rng.integers(110, 140, size=blocks).astype(np.uint8) + return codes, scales + + +def test_mxfp8_repack_layout_is_a_pure_byte_relabel(): + rng = np.random.default_rng(0) + codes, scales = _finite_fp8_pair(rng, 256, 384) + packed, group_scales = repack_fp8_block_as_mxfp8(codes, scales) + + assert packed.dtype == np.uint32 and packed.shape == (256, 96) + assert group_scales.dtype == np.uint8 and group_scales.shape == (256, 12) + # The code bytes are untouched, four to a little-endian word. + np.testing.assert_array_equal(packed.astype(" Date: Thu, 5 Nov 2026 05:03:23 +0000 Subject: [PATCH 02/74] bench: gate DSV4 headline rows on memory coming back in (row schema v2) The roadmap found the quiet gate one-sided: a row could fault gigabytes of its own working set back in and still stamp itself clean on zero pageouts and swapouts. One post-fix row passed with 245,255 page-ins (3.83 GB). collect_metric_snapshot now also reports pageins, swapins and compressor decompressions (totals and deltas). New headline rows are schema_version 2 and carry an inbound_gate. A v2 row is clean only if its outbound deltas are zero, as before, and it stayed under 4,096 page-ins, 4,096 decompressions (64 MiB each at 16 KiB pages) and zero swap-ins. vm_stat is system-wide, so zero page-ins is not attainable: the cleanest Task 4 row saw 144. The rows that faulted their working set back saw 7,296 to 304,047. A row may not declare a laxer gate than the harness's. Version 1 rows, including the committed Task 4 evidence, replay under their original rule; the quiet record's keys are unchanged. Also: - --resident-format {bf16,mxfp8} on bind-proof, run-row and series binds residents through the new opt-in mxfp8 path. Rows and the bind proof record the format, and a summary refuses rows that mix formats. - --rewarm-after-quiet (opt-in) re-runs the untimed warmup after the quiet window, so the timed window measures steady state instead of pages macOS compressed while the harness sat idle. --- benchmarks/bench_dsv4_mtp_headline.py | 96 +++++++++++-- src/ramp/benchmark/metrics.py | 14 ++ tests/test_bench_dsv4_mtp_headline_inbound.py | 136 ++++++++++++++++++ 3 files changed, 237 insertions(+), 9 deletions(-) create mode 100644 tests/test_bench_dsv4_mtp_headline_inbound.py diff --git a/benchmarks/bench_dsv4_mtp_headline.py b/benchmarks/bench_dsv4_mtp_headline.py index 9ed7d0c..ecb9b07 100644 --- a/benchmarks/bench_dsv4_mtp_headline.py +++ b/benchmarks/bench_dsv4_mtp_headline.py @@ -53,6 +53,17 @@ "GLM_SINGLE_HOST_MLX_WIRED_LIMIT_GB", ) EXPECTED_VERIFY_IMPLEMENTATION = "nax_e8p_m32n64" +#: Rows written by this harness. Version 1 rows (the committed Task 4 +#: evidence) are judged by their original outbound-only rule; version 2 rows +#: must also stay under :data:`INBOUND_GATE`. +ROW_SCHEMA_VERSION = 2 +#: Inbound-paging ceiling for a clean version 2 row, in ``vm_stat`` pages +#: (16 KiB on Apple silicon, so 4,096 pages is 64 MiB). ``vm_stat`` counts are +#: system-wide, so zero is not attainable: the cleanest Task 4 row still saw +#: 144 page-ins. The rows that faulted their working set back in saw 7,296 to +#: 304,047. Pages returning from swap are never acceptable. +INBOUND_GATE = {"max_pageins": 4096, "max_swapins": 0, "max_decompressions": 4096} +RESIDENT_FORMATS = ("bf16", "mxfp8") BENCHMARK_SOURCE_PATH = Path(__file__).resolve() RUNTIME_SOURCE_PATH = (REPO_ROOT / "src/keep/quality/dsv4_mtp_runtime.py").resolve() @@ -199,6 +210,38 @@ def _prompt(pack: Path, prompt_tokens: int) -> tuple[Any, mx.array, dict[str, An def _row_clean(row: Mapping[str, Any]) -> bool: + """Whether a row is admissible evidence under its own schema's rule.""" + + version = row.get("schema_version", 1) + if type(version) is not int or version not in (1, ROW_SCHEMA_VERSION): + raise ValueError(f"unsupported headline row schema_version {version!r}") + outbound_clean = _row_clean_outbound(row) + if version == 1: + return outbound_clean + # Validate the inbound record even when the row is already dirty, so a + # malformed v2 row cannot hide behind an outbound failure. + inbound_clean = _row_clean_inbound(row) + return outbound_clean and inbound_clean + + +def _row_clean_inbound(row: Mapping[str, Any]) -> bool: + gate = row.get("inbound_gate") + if not isinstance(gate, Mapping) or dict(gate) != INBOUND_GATE: + raise ValueError("headline inbound gate is missing or differs from the harness") + deltas = {} + for counter in ("pageins", "swapins", "decompressions"): + value = row.get(f"{counter}_delta") + if type(value) is not int or value < 0: + raise ValueError(f"headline {counter}_delta must be a nonnegative integer") + deltas[counter] = value + return ( + deltas["pageins"] <= INBOUND_GATE["max_pageins"] + and deltas["swapins"] <= INBOUND_GATE["max_swapins"] + and deltas["decompressions"] <= INBOUND_GATE["max_decompressions"] + ) + + +def _row_clean_outbound(row: Mapping[str, Any]) -> bool: quiet = row.get("quiet") if not isinstance(quiet, Mapping): raise TypeError("headline quiet record is missing or malformed") @@ -530,6 +573,11 @@ def _summarize_dsv4_mtp_rows( spec_signatures: set[str] = set() compact_stats: dict[int, dict[str, Any]] = {} prompt_authority = _validate_prompt_pack(rows[0]) + resident_formats = {str(row.get("resident_format", "bf16")) for row in rows} + if len(resident_formats) != 1 or not resident_formats <= set(RESIDENT_FORMATS): + raise ValueError( + f"headline rows mix or misstate resident formats: {sorted(resident_formats)}" + ) for row in rows: if row.get("record_type") != "dsv4_mtp_headline_row_v1": raise ValueError("unexpected DSV4 MTP headline row type") @@ -626,6 +674,7 @@ def _summarize_dsv4_mtp_rows( "clean_pairs": len(clean_pairs), "dirty_pairs": len(pairs) - len(clean_pairs), "min_clean_pairs": min_clean_pairs, + "resident_format": next(iter(resident_formats)), "orders": dict(sorted(orders.items())), "artifact_identity": representative_row["artifact_identity"], "artifact_identity_sha256": identity_sha, @@ -770,7 +819,9 @@ def _load(args: argparse.Namespace): payload_receipt=receipt, expected_payload_receipt_sha256=args.content_receipt_sha256, ) - loaded = load_authenticated_dsv4_composite(inputs) + loaded = load_authenticated_dsv4_composite( + inputs, resident_format=getattr(args, "resident_format", "bf16") + ) mx.eval(loaded.model.parameters()) return loaded @@ -800,6 +851,8 @@ def _bind_proof(args: argparse.Namespace) -> dict[str, Any]: "harness_identity": _current_harness_identity(), "harness_identity_sha256": _canonical_sha256(_current_harness_identity()), "resident_bound_count": non_vq.bound_count, + "resident_format": non_vq.resident_format, + "resident_mxfp8_repacked_tensors": len(non_vq.mxfp8_repacked_tensors), "resident_backbone_bound_count": non_vq.backbone_bound_count, "resident_mtp_bound_count": non_vq.mtp_bound_count, "resident_bound_names_sha256": _canonical_sha256( @@ -856,6 +909,15 @@ def _run_row(args: argparse.Namespace) -> dict[str, Any]: window_seconds=args.quiet_window_seconds, max_attempts=args.quiet_max_attempts, ) + rewarm = bool(getattr(args, "rewarm_after_quiet", False)) + if rewarm: + # The quiet window is idle time in which macOS may compress the + # working set the warmup just heated. Touch it again, untimed, so the + # timed window measures steady state rather than decompression. + if args.role == "baseline": + generate_dsv4_autoregressive(loaded.model, prompt, max_new_tokens=3) + else: + generate_dsv4_speculative(loaded.model, prompt, max_new_tokens=4) thermal = _thermal_snapshot() before_vm = collect_vm_stat_counts() reset_mlx_peak_memory() @@ -877,17 +939,15 @@ def _run_row(args: argparse.Namespace) -> dict[str, Any]: } ) elapsed = time.perf_counter() - started - after_vm = collect_vm_stat_counts() + # One snapshot carries the outbound and inbound deltas for the window. metrics = collect_metric_snapshot(previous_vm_stat_counts=before_vm) - pageins_delta = ( - None - if before_vm is None or after_vm is None - else int(after_vm.get("pageins", 0)) - int(before_vm.get("pageins", 0)) - ) tokens = list(result.token_ids) row = { "record_type": "dsv4_mtp_headline_row_v1", - "schema_version": 1, + "schema_version": ROW_SCHEMA_VERSION, + "inbound_gate": dict(INBOUND_GATE), + "resident_format": getattr(args, "resident_format", "bf16"), + "rewarm_after_quiet": rewarm, "fresh_process": True, "pid": os.getpid(), "ppid": os.getppid(), @@ -911,7 +971,6 @@ def _run_row(args: argparse.Namespace) -> dict[str, Any]: "speculative_stats": spec_stats, "quiet": quiet, "thermal": thermal, - "pageins_delta": pageins_delta, "wired_limit_environment_absent": True, **metrics, } @@ -980,6 +1039,9 @@ def _series(args: argparse.Namespace) -> None: str(args.quiet_window_seconds), "--quiet-max-attempts", str(args.quiet_max_attempts), + "--resident-format", + args.resident_format, + *(["--rewarm-after-quiet"] if args.rewarm_after_quiet else []), ] env = dict(os.environ) env["KEEP_TASK4_LOCK_HELD"] = "1" @@ -1031,6 +1093,20 @@ def _common_artifact_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) parser.add_argument("--resident-dir", type=Path, default=DEFAULT_RESIDENT) parser.add_argument("--vq-dir", type=Path, default=DEFAULT_VQ) + parser.add_argument( + "--resident-format", + choices=RESIDENT_FORMATS, + default="bf16", + help="bind FP8 residents decoded to bf16 (default) or as their mxfp8 source bytes", + ) + + +def _rewarm_arg(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--rewarm-after-quiet", + action="store_true", + help="re-run the untimed warmup after the quiet window, before timing", + ) def main() -> None: @@ -1055,6 +1131,7 @@ def main() -> None: row.add_argument("--role", choices=("baseline", "speculative"), required=True) row.add_argument("--quiet-window-seconds", type=float, default=5.0) row.add_argument("--quiet-max-attempts", type=int, default=3) + _rewarm_arg(row) row.add_argument("--output", type=Path, required=True) row.set_defaults(func=_run_row, heavy=True) @@ -1085,6 +1162,7 @@ def main() -> None: series.add_argument("--min-clean-pairs", type=int, default=3) series.add_argument("--quiet-window-seconds", type=float, default=5.0) series.add_argument("--quiet-max-attempts", type=int, default=3) + _rewarm_arg(series) series.add_argument("--output-dir", type=Path, required=True) series.set_defaults(func=_series, heavy=True) diff --git a/src/ramp/benchmark/metrics.py b/src/ramp/benchmark/metrics.py index ea13ab7..6f79181 100644 --- a/src/ramp/benchmark/metrics.py +++ b/src/ramp/benchmark/metrics.py @@ -13,6 +13,12 @@ MetricSnapshot = dict[str, int | None] VMStatCounts = dict[str, int] +#: ``vm_stat`` counters for memory coming *back*: file/anonymous page-ins, +#: swap-ins, and pages pulled out of the memory compressor. A row can stay at +#: zero pageouts/swapouts while faulting gigabytes of its own working set back +#: in through these, so a quiet gate that ignores them is one-sided. +INBOUND_VM_COUNTERS = ("pageins", "swapins", "decompressions") + _VM_STAT_RE = re.compile(r"^\s*(?P[^:]+):\s+(?P[0-9]+)\.?\s*$") @@ -189,4 +195,12 @@ def collect_metric_snapshot( "swapouts_total": None if vm_counts is None else vm_counts.get("swapouts"), "pageouts_delta": _delta(vm_counts, previous_vm_stat_counts, "pageouts"), "swapouts_delta": _delta(vm_counts, previous_vm_stat_counts, "swapouts"), + **{ + f"{counter}_total": None if vm_counts is None else vm_counts.get(counter) + for counter in INBOUND_VM_COUNTERS + }, + **{ + f"{counter}_delta": _delta(vm_counts, previous_vm_stat_counts, counter) + for counter in INBOUND_VM_COUNTERS + }, } diff --git a/tests/test_bench_dsv4_mtp_headline_inbound.py b/tests/test_bench_dsv4_mtp_headline_inbound.py new file mode 100644 index 0000000..07c8f1a --- /dev/null +++ b/tests/test_bench_dsv4_mtp_headline_inbound.py @@ -0,0 +1,136 @@ +"""Version 2 headline rows also gate on memory coming back in. + +The roadmap found the quiet gate one-sided: a row could fault gigabytes of its +working set back in (page-ins, swap-ins, compressor decompressions) and still +stamp itself clean on zero pageouts and swapouts. Version 2 rows close that; +version 1 rows (the committed Task 4 evidence) keep their original rule. +""" + +from __future__ import annotations + +import pytest + +import ramp.benchmark.metrics as metrics +from benchmarks import bench_dsv4_mtp_headline as benchmark + + +def _quiet(): + return { + "enabled": True, + "available": True, + "quiet": True, + "attempts": 1, + "window_seconds": 5.0, + "pageouts_delta": 0, + "swapouts_delta": 0, + } + + +def _v2_row(**changes): + row = { + "schema_version": 2, + "quiet": _quiet(), + "pageouts_delta": 0, + "swapouts_delta": 0, + "inbound_gate": dict(benchmark.INBOUND_GATE), + "pageins_delta": 144, + "swapins_delta": 0, + "decompressions_delta": 12, + } + row.update(changes) + return row + + +def test_v2_row_under_the_inbound_ceiling_is_clean(): + assert benchmark._row_clean(_v2_row()) is True + + +@pytest.mark.parametrize( + "changes", + [ + # 245,255 page-ins (3.83 GB) passed the old gate in the post-fix series. + {"pageins_delta": 245_255}, + {"pageins_delta": benchmark.INBOUND_GATE["max_pageins"] + 1}, + {"swapins_delta": 1}, + {"decompressions_delta": benchmark.INBOUND_GATE["max_decompressions"] + 1}, + ], +) +def test_v2_row_that_faulted_memory_back_in_is_dirty(changes): + assert benchmark._row_clean(_v2_row(**changes)) is False + + +def test_the_same_inbound_traffic_does_not_change_a_v1_verdict(): + """Committed version 1 evidence replays under the rule it was gated by.""" + + row = _v2_row(schema_version=1, pageins_delta=245_255) + del row["inbound_gate"], row["swapins_delta"], row["decompressions_delta"] + assert benchmark._row_clean(row) is True + del row["schema_version"] # rows that predate the field are version 1 + assert benchmark._row_clean(row) is True + + +def test_v2_outbound_failure_still_dirties_the_row(): + assert benchmark._row_clean(_v2_row(pageouts_delta=1)) is False + + +@pytest.mark.parametrize( + "changes", + [ + {"pageins_delta": None}, + {"swapins_delta": -1}, + {"decompressions_delta": 1.0}, + {"inbound_gate": None}, + # A row may not declare a laxer ceiling than the harness enforces. + {"inbound_gate": {**benchmark.INBOUND_GATE, "max_pageins": 10**9}}, + # Malformed inbound fields are caught even when outbound already failed. + {"pageouts_delta": 3, "swapins_delta": None}, + ], +) +def test_v2_inbound_record_is_validated_not_trusted(changes): + with pytest.raises(ValueError, match="inbound|_delta"): + benchmark._row_clean(_v2_row(**changes)) + + +@pytest.mark.parametrize("version", [0, 3, "2", True]) +def test_unknown_row_schema_versions_are_refused(version): + with pytest.raises(ValueError, match="schema_version"): + benchmark._row_clean(_v2_row(schema_version=version)) + + +def test_metric_snapshot_reports_inbound_counters(monkeypatch): + monkeypatch.setattr( + metrics, + "collect_vm_stat_counts", + lambda: { + "pageouts": 5, + "swapouts": 6, + "pageins": 1_000, + "swapins": 7, + "decompressions": 50, + }, + ) + monkeypatch.setattr(metrics, "_mlx_memory_value", lambda _name: 0) + snapshot = metrics.collect_metric_snapshot( + previous_vm_stat_counts={ + "pageouts": 5, + "swapouts": 6, + "pageins": 856, + "swapins": 7, + "decompressions": 38, + } + ) + assert snapshot["pageins_delta"] == 144 + assert snapshot["swapins_delta"] == 0 + assert snapshot["decompressions_delta"] == 12 + assert snapshot["pageins_total"] == 1_000 + assert set(metrics.INBOUND_VM_COUNTERS) == {"pageins", "swapins", "decompressions"} + + +def test_vm_stat_parser_reads_the_inbound_lines(): + counts = metrics.parse_vm_stat_counts( + "Mach Virtual Memory Statistics: (page size of 16384 bytes)\n" + "Pageins: 123.\n" + "Swapins: 4.\n" + "Decompressions: 56.\n" + ) + assert counts == {"pageins": 123, "swapins": 4, "decompressions": 56} From bd7fd54d2aaff24b03210f9a81be28672c1d4545 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:10:39 +0000 Subject: [PATCH 03/74] keep: REAP expert saliency from the real gate, collected in the calibration pass keep.quality.reap scores each routed expert by REAP's criterion, the mean over the slots routed to it of gate weight times the L2 norm of its output before gating. The inputs come from what KEEP's teacher runner already computes: the model's own gate (bias-corrected selection, sqrtsoftplus weights, renormalisation) and the per-route expert outputs. Upstream REAP's code instead takes softmax of raw logits, picks top-k without the correction bias, and runs every expert on every token. Here the cost is a norm and five scatter-adds per layer, folded into float64 once per layer like the imatrix. - layer_major_prefill takes an optional ReapAccumulator; the forward stays bit-identical (tested alone and alongside the imatrix). - Calibration sessions now also write reap__* arrays. Capture validation checks them and requires their route counts to match the imatrix's. The imatrix finalizer ignores them, so existing runs still finalize. - finalize_dsv4_reap_stats sums them, and refuses if any session lacks them rather than silently under-scoring what that session routed to. - produce_dsv4_teacher_cache.py finalize-reap writes reap-stats.npz. reap-select picks a uniform survivor set and reports per-layer Jaccard of the pruned set against route frequency. That is the plan's kill test: Jaccard >= 0.9 means REAP adds nothing over the counts KEEP already had. - expert_saliency offers reap, frequency, weighted_frequency and ean_mean, plus optional shrinkage of rare experts toward the layer mean. - super_expert_candidates applies REAP's outlier rule to max |f|, so negative outliers are not invisible as they are upstream. - prune_checkpoint_plan maps checkpoint names to a uniform prune: it renumbers kept experts and slices gate weight, bias and bias_vl (V4.1's image-token bias, which must not be left at the old width). It refuses hash-routed blocks (tid2eid), which need a remapping policy. A test checks that the sliced gate picks the same experts with the same renormalised weights as the full gate with pruned experts masked out of selection. --- benchmarks/produce_dsv4_teacher_cache.py | 87 +++++ src/keep/quality/dsv4_teacher_runner.py | 95 ++++- src/keep/quality/reap.py | 475 +++++++++++++++++++++++ tests/test_dsv4_teacher_runner.py | 117 ++++++ tests/test_reap.py | 333 ++++++++++++++++ 5 files changed, 1105 insertions(+), 2 deletions(-) create mode 100644 src/keep/quality/reap.py create mode 100644 tests/test_reap.py diff --git a/benchmarks/produce_dsv4_teacher_cache.py b/benchmarks/produce_dsv4_teacher_cache.py index 53e718e..75549b6 100644 --- a/benchmarks/produce_dsv4_teacher_cache.py +++ b/benchmarks/produce_dsv4_teacher_cache.py @@ -237,6 +237,67 @@ def _cmd_finalize(opts: argparse.Namespace) -> int: return 0 +def _cmd_finalize_reap(opts: argparse.Namespace) -> int: + from keep.quality.dsv4_teacher_runner import finalize_dsv4_reap_stats + + report = finalize_dsv4_reap_stats( + Path(opts.out_dir).expanduser(), + output=Path(opts.output).expanduser() if opts.output else None, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +def _cmd_reap_select(opts: argparse.Namespace) -> int: + """Pick survivors and run the REAP-vs-baseline kill test on the same stats.""" + + import numpy as np + + from keep.quality.reap import ( + ReapStats, + expert_saliency, + pruned_set_jaccard, + select_survivors, + super_expert_candidates, + write_survivors, + ) + + stats = ReapStats.load(Path(opts.stats).expanduser()) + chosen = select_survivors( + expert_saliency(stats, opts.criterion, prior_slots=opts.prior_slots), opts.keep + ) + baseline = select_survivors(expert_saliency(stats, opts.compare), opts.keep) + jaccard = pruned_set_jaccard(chosen, baseline, num_experts=stats.num_experts) + report = { + "record_type": "dsv4_reap_select", + "criterion": opts.criterion, + "prior_slots": opts.prior_slots, + "keep": opts.keep, + "num_experts": stats.num_experts, + "sessions": len(stats.sessions), + "compare": opts.compare, + "pruned_set_jaccard_vs_compare": { + str(int(layer)): round(float(value), 4) + for layer, value in zip(stats.layers, jaccard) + }, + "median_jaccard": float(np.median(jaccard)), + # Kill test from the plan: if REAP's pruned set matches frequency's + # at Jaccard >= 0.9, REAP adds nothing over the counts KEEP already has. + "reap_adds_little": bool(np.median(jaccard) >= 0.9), + "never_routed_experts": int((stats.route_count == 0).sum()), + "super_expert_candidates": super_expert_candidates(stats), + } + if opts.output: + write_survivors( + Path(opts.output).expanduser(), + chosen, + layers=stats.layers.tolist(), + meta={k: v for k, v in report.items() if k != "record_type"}, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + def _cmd_plan(opts: argparse.Namespace) -> int: from keep.quality.dsv4_teacher_runner import ( build_dsv4_expert_span_index, @@ -347,6 +408,32 @@ def build_parser() -> argparse.ArgumentParser: finalize.add_argument("--prompt-set", default="dsv4_teich_calibration_v1") finalize.set_defaults(func=_cmd_finalize) + finalize_reap = verbs.add_parser( + "finalize-reap", help="sum calibration sessions' REAP expert statistics" + ) + finalize_reap.add_argument("--out-dir", required=True) + finalize_reap.add_argument("--output", help="default: /reap-stats.npz") + finalize_reap.set_defaults(func=_cmd_finalize_reap) + + select = verbs.add_parser( + "reap-select", help="choose surviving experts and compare criteria" + ) + select.add_argument("--stats", required=True, help="reap-stats.npz from finalize-reap") + select.add_argument("--keep", type=int, required=True, help="experts kept per layer") + select.add_argument( + "--criterion", + default="reap", + choices=("reap", "frequency", "weighted_frequency", "ean_mean"), + ) + select.add_argument( + "--compare", + default="frequency", + choices=("reap", "frequency", "weighted_frequency", "ean_mean"), + ) + select.add_argument("--prior-slots", type=float, default=0.0) + select.add_argument("--output", help="write the survivor table as JSON") + select.set_defaults(func=_cmd_reap_select) + plan = verbs.add_parser("plan", help="print the streaming plan and byte budget") plan.add_argument("--checkpoint", default=str(DEFAULT_CHECKPOINT)) plan.add_argument("--pack", default=str(DEFAULT_PACK)) diff --git a/src/keep/quality/dsv4_teacher_runner.py b/src/keep/quality/dsv4_teacher_runner.py index 9979193..aa12438 100644 --- a/src/keep/quality/dsv4_teacher_runner.py +++ b/src/keep/quality/dsv4_teacher_runner.py @@ -92,6 +92,7 @@ "build_dsv4_expert_span_index", "calibration_session_arrays", "finalize_dsv4_calibration_imatrix", + "finalize_dsv4_reap_stats", "layer_major_prefill", "load_dsv4_streaming_workload", "load_dsv4_teich_pack", @@ -1162,6 +1163,7 @@ def layer_major_prefill( on_layer: Callable[[int, int], None] | None = None, release_after_layer: bool = True, dspark_taps: dict[int, list[Any]] | None = None, + reap: Any | None = None, ) -> list[Any]: """Prefill the whole backbone layer-outer, chunk-inner. @@ -1182,6 +1184,12 @@ def layer_major_prefill( :meth:`DeepseekV4FlashBackbone.__call__`. Layer-major order is why this has to be a dict of lists rather than one concatenated array: the taps for layer 40 are complete before layer 41 has run at all. + + ``reap`` is an optional :class:`keep.quality.reap.ReapAccumulator`. It + takes the model's own gate weights and each routed expert's output before + gating, so it scores experts against the routing the model actually uses. + Like the imatrix it reads tensors the forward already computes and leaves + the forward bit-identical. """ import mlx.core as mx @@ -1284,7 +1292,7 @@ def down_hook(activated, expert_ids, order, _capture=capture): ) mask = masks[chunk_index] - if accumulator is None: + if accumulator is None and reap is None: hidden[chunk_index] = layer(h, mask, layer_cache, piece) if layer_index in tap_layers: tap = _dspark_tap(hidden[chunk_index]) @@ -1316,6 +1324,17 @@ def down_hook(activated, expert_ids, order, _capture=capture): flat_inds = inds.reshape(-1, inds.shape[-1]) flat_scores = scores.reshape(-1, scores.shape[-1]).astype(mx.float32) + if reap is not None: + reap.add_chunk( + layer=layer_index, + expert_ids=flat_inds, + gate_weights=flat_scores, + expert_outputs=routed.reshape(-1, *routed.shape[-2:]), + ) + if accumulator is None: + mx.eval(hidden[chunk_index]) + reap.eval_pending() + continue if collect_hidden: rows = moe_input.reshape(-1, moe_input.shape[-1]).astype(mx.float32) accumulator.add_chunk( @@ -1344,6 +1363,8 @@ def down_hook(activated, expert_ids, order, _capture=capture): ) mx.eval(hidden[chunk_index]) accumulator.eval_pending() + if reap is not None: + reap.eval_pending() switch_mlp.down_hook = None layer.ffn.switch_mlp = None @@ -1352,6 +1373,8 @@ def down_hook(activated, expert_ids, order, _capture=capture): expert_provider.release(layer_index) if accumulator is not None: accumulator.flush_device() + if reap is not None: + reap.flush_device() mx.clear_cache() if on_layer is not None: on_layer(layer_index, n_chunks) @@ -1933,6 +1956,19 @@ def validate_dsv4_calibration_capture( raise ValueError( f"route_count__{space} rows must sum to total_route_count__{space}" ) + if any(str(name).startswith("reap__") for name in capture): + from keep.quality.reap import ReapStats + + reap = ReapStats.from_arrays(capture) # validates shapes and invariants + if reap.layers.tolist() != stored_layers.tolist(): + raise ValueError("calibration REAP layer set drifted") + if reap.num_experts != int(num_experts): + raise ValueError("calibration REAP expert count drifted") + hidden_counts = capture.get("route_count__hidden") + if hidden_counts is not None and not np.array_equal( + np.asarray(hidden_counts), reap.route_count + ): + raise ValueError("REAP route counts disagree with the imatrix route counts") def calibration_session_arrays( @@ -2665,8 +2701,12 @@ def _produce_sessions( started = time.perf_counter() token_ids = mx.array(session.token_ids.astype(np.int64)[None, :]) accumulator = None + reap = None collect: tuple[str, ...] = () if mode == "calibration": + from keep.quality.reap import ReapAccumulator + + reap = ReapAccumulator(layers=layers, num_experts=num_experts) accumulator = _ImatrixAccumulator( layers=layers, num_experts=num_experts, @@ -2696,10 +2736,14 @@ def on_layer(layer_index: int, n_chunks: int, _s=session, _i=index) -> None: collect_spaces=collect, on_layer=on_layer, dspark_taps=taps, + reap=reap, ) if mode == "calibration": - arrays = calibration_session_arrays(accumulator) + arrays = { + **calibration_session_arrays(accumulator), + **reap.stats(session=session.prompt_id).as_arrays(), + } validate_dsv4_calibration_capture( arrays, layers=layers, num_experts=num_experts ) @@ -2797,6 +2841,53 @@ def _atomic_json(path: Path, payload: Any) -> None: # --------------------------------------------------------------------------- +def finalize_dsv4_reap_stats( + out_dir: str | Path, *, output: str | Path | None = None +) -> dict[str, Any]: + """Sum every calibration session's REAP statistics into one file. + + Fails closed if any calibration session lacks them (for example a session + produced before REAP collection existed), because a saliency missing some + sessions would silently under-score whatever those sessions routed to. + """ + + from keep.quality.reap import REAP_ARRAY_PREFIX, ReapStats + + root = Path(out_dir) + session_files = sorted((root / "sessions").glob("*.npz")) + if not session_files: + raise ValueError(f"no session outputs under {root / 'sessions'}") + total: ReapStats | None = None + missing: list[str] = [] + for path in session_files: + with np.load(path, allow_pickle=False) as archive: + capture = {name: archive[name] for name in archive.files} + if _identity_text(capture, "record_type") != "dsv4_teacher_calibration_v1": + raise ValueError(f"{path} is not a calibration capture") + if not any(name.startswith(REAP_ARRAY_PREFIX) for name in capture): + missing.append(path.name) + continue + stats = ReapStats.from_arrays(capture, sessions=(_identity_text(capture, "prompt_id"),)) + total = stats if total is None else total.merged(stats) + if missing: + raise ValueError( + f"{len(missing)} calibration sessions have no REAP statistics " + f"(rerun them): {missing[:5]}" + ) + assert total is not None + target = Path(output) if output is not None else root / "reap-stats.npz" + total.save(target) + return { + "record_type": "dsv4_reap_stats_finalize", + "session_count": len(session_files), + "layers": total.layers.tolist(), + "num_experts": total.num_experts, + "routed_slots": int(total.routed_slots.sum()), + "never_routed_experts": int((total.route_count == 0).sum()), + "output": str(target), + } + + def finalize_dsv4_calibration_imatrix( out_dir: str | Path, *, diff --git a/src/keep/quality/reap.py b/src/keep/quality/reap.py new file mode 100644 index 0000000..8e7f0d0 --- /dev/null +++ b/src/keep/quality/reap.py @@ -0,0 +1,475 @@ +"""REAP expert saliency from the real routed forward, and the prune it drives. + +REAP (Router-weighted Expert Activation Pruning) scores expert ``i`` by the +mean, over the tokens actually routed to it, of ``g * ||f_i(x)||_2``: the gate +weight that combined its output times the L2 norm of that output before +gating. Experts with the lowest score are dropped and the router re-selects +top-k among the survivors. + +This module computes that score from what KEEP's teacher runner already has +in hand -- the model's own gate (bias-corrected selection, the family's own +weight function and renormalisation) and the per-route expert outputs -- so it +costs a norm and a few scatter-adds per layer instead of REAP's dense pass, +which runs every expert on every token. Two differences from the upstream +implementation are deliberate: + +- **The real gate.** Upstream computes ``softmax`` of raw router logits and + picks top-k without the correction bias. For a sqrtsoftplus/noaux router + (DeepSeek-V4, V4.1) that is a different routing distribution; the score + here uses the weights the model actually used. +- **Routed tokens only.** Saliency never needs an unrouted expert's output. + +The statistics also support the baselines a kill test compares against +(route frequency, gate-weighted frequency, mean output norm) and the +"super-expert" outlier screen, which here uses ``max |f|`` so negative +outliers are not invisible. + +Pruning is uniform: one surviving expert count for every layer, because RAMP +binds a single ``n_routed_experts``. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +import numpy as np + +__all__ = [ + "REAP_ARRAY_PREFIX", + "ReapAccumulator", + "ReapStats", + "SALIENCY_CRITERIA", + "expert_saliency", + "prune_checkpoint_plan", + "pruned_set_jaccard", + "select_survivors", + "super_expert_candidates", + "write_survivors", +] + +REAP_ARRAY_PREFIX = "reap__" +SALIENCY_CRITERIA = ("reap", "frequency", "weighted_frequency", "ean_mean") +Criterion = Literal["reap", "frequency", "weighted_frequency", "ean_mean"] + +_STAT_FIELDS = ( + "route_count", # int64 [L, E]: routed slots per expert + "gate_sum", # float64 [L, E]: sum of gate weights + "gate_norm_sum", # float64 [L, E]: sum of g * ||f|| (the REAP numerator) + "norm_sum", # float64 [L, E]: sum of ||f|| + "norm_max", # float64 [L, E]: max ||f|| over routed slots + "abs_max", # float64 [L, E]: max over routed slots of max_h |f_h| +) + + +@dataclass +class ReapStats: + """Summed REAP statistics for one or more sessions.""" + + layers: np.ndarray # int64 [L] + num_experts: int + route_count: np.ndarray + gate_sum: np.ndarray + gate_norm_sum: np.ndarray + norm_sum: np.ndarray + norm_max: np.ndarray + abs_max: np.ndarray + routed_slots: np.ndarray # int64 [L]: total routed slots per layer + sessions: tuple[str, ...] = field(default_factory=tuple) + + def __post_init__(self) -> None: + self.layers = np.asarray(self.layers, dtype=np.int64) + n = self.layers.size + shape = (n, int(self.num_experts)) + self.route_count = np.asarray(self.route_count, dtype=np.int64) + for name in _STAT_FIELDS[1:]: + setattr(self, name, np.asarray(getattr(self, name), dtype=np.float64)) + self.routed_slots = np.asarray(self.routed_slots, dtype=np.int64) + for name in _STAT_FIELDS: + value = getattr(self, name) + if value.shape != shape: + raise ValueError(f"REAP {name} has shape {value.shape}, expected {shape}") + if not np.all(np.isfinite(value)) or np.any(value < 0): + raise ValueError(f"REAP {name} must be finite and non-negative") + if self.routed_slots.shape != (n,): + raise ValueError("REAP routed_slots must be one count per layer") + if np.any(self.route_count.sum(axis=1) != self.routed_slots): + raise ValueError("REAP route counts do not sum to the routed slots") + if np.any((self.route_count == 0) & (self.gate_norm_sum != 0)): + raise ValueError("REAP has gate mass on an expert with no routed slots") + + # -- persistence ------------------------------------------------------- + + def as_arrays(self) -> dict[str, np.ndarray]: + arrays = { + f"{REAP_ARRAY_PREFIX}layers": self.layers, + f"{REAP_ARRAY_PREFIX}num_experts": np.asarray(self.num_experts, dtype=np.int64), + f"{REAP_ARRAY_PREFIX}routed_slots": self.routed_slots, + } + for name in _STAT_FIELDS: + arrays[f"{REAP_ARRAY_PREFIX}{name}"] = getattr(self, name) + return arrays + + @classmethod + def from_arrays( + cls, arrays: Mapping[str, Any], *, sessions: Sequence[str] = () + ) -> "ReapStats": + key = REAP_ARRAY_PREFIX.__add__ + missing = [ + name + for name in ("layers", "num_experts", "routed_slots", *_STAT_FIELDS) + if key(name) not in arrays + ] + if missing: + raise KeyError("REAP arrays missing: " + ", ".join(missing)) + return cls( + layers=np.asarray(arrays[key("layers")]), + num_experts=int(np.asarray(arrays[key("num_experts")]).reshape(())), + routed_slots=np.asarray(arrays[key("routed_slots")]), + sessions=tuple(sessions), + **{name: np.asarray(arrays[key(name)]) for name in _STAT_FIELDS}, + ) + + def merged(self, other: "ReapStats") -> "ReapStats": + """Sum two disjoint sets of sessions over the same layers and experts.""" + + if not np.array_equal(self.layers, other.layers) or self.num_experts != other.num_experts: + raise ValueError("REAP stats cover different layers or expert counts") + overlap = set(self.sessions) & set(other.sessions) + if overlap: + raise ValueError(f"REAP stats would double-count sessions: {sorted(overlap)[:5]}") + return ReapStats( + layers=self.layers, + num_experts=self.num_experts, + route_count=self.route_count + other.route_count, + gate_sum=self.gate_sum + other.gate_sum, + gate_norm_sum=self.gate_norm_sum + other.gate_norm_sum, + norm_sum=self.norm_sum + other.norm_sum, + norm_max=np.maximum(self.norm_max, other.norm_max), + abs_max=np.maximum(self.abs_max, other.abs_max), + routed_slots=self.routed_slots + other.routed_slots, + sessions=self.sessions + other.sessions, + ) + + def save(self, path: str | Path) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.with_name(target.name + ".tmp.npz") + np.savez(tmp, **self.as_arrays(), **{f"{REAP_ARRAY_PREFIX}sessions": np.asarray(self.sessions, dtype=str)}) + tmp.replace(target) + + @classmethod + def load(cls, path: str | Path) -> "ReapStats": + with np.load(path, allow_pickle=False) as archive: + arrays = {name: archive[name] for name in archive.files} + sessions = tuple(str(s) for s in arrays.pop(f"{REAP_ARRAY_PREFIX}sessions", np.asarray([], dtype=str))) + return cls.from_arrays(arrays, sessions=sessions) + + +class ReapAccumulator: + """Device scatter-add of REAP statistics; folds into float64 once per layer. + + Same design as the teacher runner's imatrix accumulator: float32 sums on + device for one layer's chunks, then a float64 host fold, so cross-chunk and + cross-session accumulation carries no float32 drift beyond one layer. + """ + + def __init__(self, *, layers: Sequence[int], num_experts: int) -> None: + self.layers = tuple(int(layer) for layer in layers) + self.num_experts = int(num_experts) + self._row = {layer: row for row, layer in enumerate(self.layers)} + shape = (len(self.layers), self.num_experts) + self._host = {name: np.zeros(shape, dtype=np.float64) for name in _STAT_FIELDS} + self._routed_slots = np.zeros(len(self.layers), dtype=np.int64) + self._device: dict[int, dict[str, Any]] = {} + + def add_chunk( + self, *, layer: int, expert_ids: Any, gate_weights: Any, expert_outputs: Any + ) -> None: + """Add one chunk's routed slots. + + ``expert_ids`` and ``gate_weights`` are ``[rows, top_k]``; + ``expert_outputs`` is ``[rows, top_k, hidden]``, each expert's output + for that slot *before* the gate weight multiplies it. + """ + + import mlx.core as mx + + if layer not in self._row: + raise KeyError(f"layer {layer} is not tracked by this accumulator") + if expert_outputs.ndim != 3 or tuple(expert_outputs.shape[:2]) != tuple(expert_ids.shape): + raise ValueError( + f"expert outputs {tuple(expert_outputs.shape)} do not match routes " + f"{tuple(expert_ids.shape)}" + ) + outputs = expert_outputs.astype(mx.float32) + norms = mx.sqrt(mx.sum(outputs * outputs, axis=-1)).reshape(-1) + abs_max = mx.max(mx.abs(outputs), axis=-1).reshape(-1) + ids = expert_ids.reshape(-1).astype(mx.int32) + gates = gate_weights.reshape(-1).astype(mx.float32) + bucket = self._device.get(layer) + if bucket is None: + zeros = lambda: mx.zeros((self.num_experts,), dtype=mx.float32) # noqa: E731 + bucket = {name: zeros() for name in _STAT_FIELDS} + bucket["slots"] = 0 + self._device[layer] = bucket + bucket["route_count"] = bucket["route_count"].at[ids].add(mx.ones_like(norms)) + bucket["gate_sum"] = bucket["gate_sum"].at[ids].add(gates) + bucket["gate_norm_sum"] = bucket["gate_norm_sum"].at[ids].add(gates * norms) + bucket["norm_sum"] = bucket["norm_sum"].at[ids].add(norms) + bucket["norm_max"] = bucket["norm_max"].at[ids].maximum(norms) + bucket["abs_max"] = bucket["abs_max"].at[ids].maximum(abs_max) + bucket["slots"] += int(ids.size) + + def eval_pending(self) -> None: + import mlx.core as mx + + for bucket in self._device.values(): + mx.eval([bucket[name] for name in _STAT_FIELDS]) + + def flush_device(self) -> None: + import mlx.core as mx + + for layer, bucket in self._device.items(): + mx.eval([bucket[name] for name in _STAT_FIELDS]) + row = self._row[layer] + for name in ("gate_sum", "gate_norm_sum", "norm_sum"): + self._host[name][row] += np.asarray(bucket[name], dtype=np.float64) + # float32 scatter-added ones are exact small integers; rint makes a + # future non-integral weighting fail loudly at the int cast below. + self._host["route_count"][row] += np.rint(np.asarray(bucket["route_count"], dtype=np.float64)) + for name in ("norm_max", "abs_max"): + self._host[name][row] = np.maximum( + self._host[name][row], np.asarray(bucket[name], dtype=np.float64) + ) + self._routed_slots[row] += int(bucket["slots"]) + self._device.clear() + + def stats(self, *, session: str | None = None) -> ReapStats: + self.flush_device() + return ReapStats( + layers=np.asarray(self.layers, dtype=np.int64), + num_experts=self.num_experts, + route_count=self._host["route_count"].astype(np.int64), + gate_sum=self._host["gate_sum"], + gate_norm_sum=self._host["gate_norm_sum"], + norm_sum=self._host["norm_sum"], + norm_max=self._host["norm_max"], + abs_max=self._host["abs_max"], + routed_slots=self._routed_slots.copy(), + sessions=() if session is None else (session,), + ) + + def as_arrays(self) -> dict[str, np.ndarray]: + return self.stats().as_arrays() + + +def expert_saliency( + stats: ReapStats, criterion: Criterion = "reap", *, prior_slots: float = 0.0 +) -> np.ndarray: + """Per-(layer, expert) saliency under ``criterion``; higher means keep. + + - ``reap``: mean of ``g * ||f||`` over routed slots (REAP's score). + - ``frequency``: routed slot count. + - ``weighted_frequency``: sum of gate weights. + - ``ean_mean``: mean output norm over routed slots, ignoring the gate. + + An expert with no routed slots scores 0 under every criterion, so it is + pruned first, as upstream does. ``prior_slots > 0`` shrinks the mean-type + scores toward the layer mean as if each expert had seen that many extra + average slots, which damps the noise of rarely routed experts; 0 (the + default) is REAP exactly. + """ + + if criterion not in SALIENCY_CRITERIA: + raise ValueError(f"criterion must be one of {SALIENCY_CRITERIA}, got {criterion!r}") + if prior_slots < 0: + raise ValueError("prior_slots must be non-negative") + count = stats.route_count.astype(np.float64) + if criterion == "frequency": + return count + if criterion == "weighted_frequency": + return stats.gate_sum.copy() + numerator = stats.gate_norm_sum if criterion == "reap" else stats.norm_sum + with np.errstate(invalid="ignore", divide="ignore"): + if prior_slots == 0: + return np.where(count > 0, numerator / np.maximum(count, 1.0), 0.0) + layer_mean = numerator.sum(axis=1, keepdims=True) / np.maximum( + count.sum(axis=1, keepdims=True), 1.0 + ) + shrunk = (numerator + prior_slots * layer_mean) / (count + prior_slots) + return np.where(count > 0, shrunk, 0.0) + + +def select_survivors(saliency: np.ndarray, keep: int) -> np.ndarray: + """Keep the ``keep`` most salient experts in every layer. + + Returns ``int64 [L, keep]`` expert ids, ascending within each layer, which + is the order the sliced router rows and renumbered experts follow. Ties + break toward the lower expert id, so the result is deterministic. + """ + + saliency = np.asarray(saliency, dtype=np.float64) + if saliency.ndim != 2: + raise ValueError("saliency must be [layers, experts]") + layers, experts = saliency.shape + if not 0 < int(keep) <= experts: + raise ValueError(f"keep must be in 1..{experts}, got {keep}") + if not np.all(np.isfinite(saliency)): + raise ValueError("saliency must be finite") + survivors = np.empty((layers, int(keep)), dtype=np.int64) + ids = np.arange(experts) + for row in range(layers): + order = np.lexsort((ids, -saliency[row])) # by saliency desc, then id asc + survivors[row] = np.sort(order[: int(keep)]) + return survivors + + +def pruned_set_jaccard(a: np.ndarray, b: np.ndarray, *, num_experts: int) -> np.ndarray: + """Per-layer Jaccard index of the *pruned* sets implied by two survivor lists.""" + + a = np.asarray(a) + b = np.asarray(b) + if a.shape != b.shape or a.ndim != 2: + raise ValueError("survivor arrays must share a [layers, keep] shape") + out = np.empty(a.shape[0], dtype=np.float64) + universe = set(range(int(num_experts))) + for row in range(a.shape[0]): + pruned_a = universe - set(a[row].tolist()) + pruned_b = universe - set(b[row].tolist()) + union = pruned_a | pruned_b + out[row] = 1.0 if not union else len(pruned_a & pruned_b) / len(union) + return out + + +def super_expert_candidates( + stats: ReapStats, + *, + percentile: float = 99.5, + fraction_of_max: float = 0.1, + layer_fraction: float = 0.75, +) -> list[tuple[int, int]]: + """``(layer, expert)`` pairs whose largest routed output is an outlier. + + REAP's rule on ``max |f|`` rather than the signed maximum: an expert is a + candidate if its peak exceeds both the ``percentile`` of all peaks and + ``fraction_of_max`` of the global peak, within the first ``layer_fraction`` + of layers. Upstream starts its signed maximum at 0, so negative outliers + never register; this screen sees them. + """ + + peaks = stats.abs_max + if peaks.size == 0: + return [] + threshold = max(float(np.percentile(peaks, percentile)), float(peaks.max()) * fraction_of_max) + limit = int(np.ceil(len(stats.layers) * layer_fraction)) + hits = np.argwhere(peaks[:limit] > threshold) + return [(int(stats.layers[row]), int(expert)) for row, expert in hits] + + +_ROUTER_RE = re.compile(r"^(?P(?:layers|mtp)\.(?P\d+))\.ffn\.gate\.(?Pweight|bias|bias_vl|tid2eid)$") +_EXPERT_RE = re.compile( + r"^(?P(?:layers|mtp)\.(?P\d+))\.ffn\.experts\.(?P\d+)\.(?P.+)$" +) + + +def prune_checkpoint_plan( + names: Iterable[str], + survivors: Mapping[int, Sequence[int]], + *, + prefix: str = "layers", +) -> dict[str, Any]: + """Plan a uniform prune over DeepSeek-V4-style checkpoint tensor names. + + ``survivors`` maps a block index under ``prefix`` (``layers`` for the + backbone, ``mtp`` for drafter stages) to the expert ids it keeps. Returns: + + - ``rename``: ``{source_name: target_name}`` for every kept routed-expert + tensor, renumbered ``0..keep-1`` in ascending source order; + - ``slice_rows``: ``{router_tensor_name: [kept ids]}`` for the gate's + ``weight``, ``bias`` and ``bias_vl`` (the image-token bias V4.1 adds, + which must be sliced with the rest or vision tokens route to the wrong + experts); + - ``drop``: expert tensors of pruned experts; + - ``untouched``: every other name, including blocks not in ``survivors``. + + Hash-routed blocks (``gate.tid2eid``) are refused: their token-to-expert + table names fixed experts and needs a remapping policy this plan does not + invent. DeepSeek-V4.1 has none; V4-Flash has three. + """ + + keep_sets: dict[int, list[int]] = {} + width = None + for block, kept in survivors.items(): + kept_sorted = sorted(int(e) for e in kept) + if len(set(kept_sorted)) != len(kept_sorted): + raise ValueError(f"block {block} keeps a duplicate expert") + if width is None: + width = len(kept_sorted) + elif len(kept_sorted) != width: + raise ValueError("every pruned block must keep the same expert count") + keep_sets[int(block)] = kept_sorted + + rename: dict[str, str] = {} + slice_rows: dict[str, list[int]] = {} + drop: list[str] = [] + untouched: list[str] = [] + seen_experts: dict[int, set[int]] = {block: set() for block in keep_sets} + for name in sorted(names): + router = _ROUTER_RE.match(name) + expert = _EXPERT_RE.match(name) + match = router or expert + if match is None or not match.group("prefix").startswith(prefix + "."): + untouched.append(name) + continue + block = int(match.group("idx")) + if block not in keep_sets: + untouched.append(name) + continue + kept = keep_sets[block] + if router is not None: + if router.group("leaf") == "tid2eid": + raise NotImplementedError( + f"{name}: hash-routed block {block} needs a token-to-expert " + "remapping policy before it can be pruned" + ) + slice_rows[name] = list(kept) + continue + old = int(expert.group("expert")) + seen_experts[block].add(old) + if old in kept: + new = kept.index(old) + rename[name] = f"{expert.group('prefix')}.ffn.experts.{new}.{expert.group('rest')}" + else: + drop.append(name) + for block, kept in keep_sets.items(): + absent = sorted(set(kept) - seen_experts[block]) + if absent: + raise ValueError(f"block {block} keeps experts with no tensors: {absent[:10]}") + return { + "record_type": "keep_reap_prune_plan_v1", + "prefix": prefix, + "keep": width, + "survivors": {str(block): kept for block, kept in sorted(keep_sets.items())}, + "rename": rename, + "slice_rows": slice_rows, + "drop": drop, + "untouched": untouched, + } + + +def write_survivors(path: str | Path, survivors: np.ndarray, *, layers: Sequence[int], meta: Mapping[str, Any]) -> None: + """Persist a survivor table with the settings that produced it.""" + + payload = { + "record_type": "keep_reap_survivors_v1", + "layers": [int(layer) for layer in layers], + "keep": int(survivors.shape[1]), + "survivors": {str(int(layer)): survivors[row].tolist() for row, layer in enumerate(layers)}, + **dict(meta), + } + Path(path).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") diff --git a/tests/test_dsv4_teacher_runner.py b/tests/test_dsv4_teacher_runner.py index 6b725b7..f403fc8 100644 --- a/tests/test_dsv4_teacher_runner.py +++ b/tests/test_dsv4_teacher_runner.py @@ -1781,6 +1781,123 @@ def test_gate_and_up_share_the_hidden_space_vector(self, tmp_path): assert gate["importance_sum"].shape == (args.hidden_size,) +class TestReapCollection: + """REAP saliency rides the calibration pass without touching the forward.""" + + def _prefill(self, *, reap=None, accumulator=None): + args, model = _tiny_model() + experts = _tiny_experts(args) + ids = mx.array(np.random.default_rng(5).integers(0, args.vocab_size, (1, 48))) + out = layer_major_prefill( + model, + ids, + chunk=16, + expert_provider=InMemoryExpertProvider(experts), + accumulator=accumulator, + collect_spaces=IMATRIX_SPACES if accumulator is not None else (), + release_after_layer=False, + reap=reap, + ) + return args, out + + def test_reap_alone_and_with_the_imatrix_leave_the_forward_unchanged(self): + from keep.quality.reap import ReapAccumulator + + args, plain = self._prefill() + layers = range(args.num_hidden_layers) + _, reap_only = self._prefill( + reap=ReapAccumulator(layers=layers, num_experts=args.n_routed_experts) + ) + _, both = self._prefill( + reap=ReapAccumulator(layers=layers, num_experts=args.n_routed_experts), + accumulator=_ImatrixAccumulator( + layers=layers, + num_experts=args.n_routed_experts, + dims={"hidden": args.hidden_size, "down": args.moe_intermediate_size}, + ), + ) + for a, b, c in zip(plain, reap_only, both): + assert mx.array_equal(a, b) and mx.array_equal(a, c) + + def test_reap_sees_the_same_routes_and_gate_weights_as_the_imatrix(self): + from keep.quality.reap import ReapAccumulator + + args, _ = _tiny_model() + layers = list(range(args.num_hidden_layers)) + reap = ReapAccumulator(layers=layers, num_experts=args.n_routed_experts) + imatrix = _ImatrixAccumulator( + layers=layers, + num_experts=args.n_routed_experts, + dims={"hidden": args.hidden_size, "down": args.moe_intermediate_size}, + ) + self._prefill(reap=reap, accumulator=imatrix) + stats = reap.stats() + arrays = imatrix.as_arrays() + np.testing.assert_array_equal(stats.route_count, arrays["route_count__hidden"]) + np.testing.assert_allclose( + stats.gate_sum, arrays["affinity_score_sum__hidden"], rtol=1e-6 + ) + assert np.all(stats.gate_norm_sum[stats.route_count > 0] > 0) + validate_dsv4_calibration_capture( + {**arrays, **stats.as_arrays()}, + layers=layers, + num_experts=args.n_routed_experts, + ) + + def test_capture_validation_refuses_reap_counts_that_disagree(self): + from keep.quality.reap import ReapAccumulator + + args, _ = _tiny_model() + layers = list(range(args.num_hidden_layers)) + reap = ReapAccumulator(layers=layers, num_experts=args.n_routed_experts) + imatrix = _ImatrixAccumulator( + layers=layers, + num_experts=args.n_routed_experts, + dims={"hidden": args.hidden_size, "down": args.moe_intermediate_size}, + ) + self._prefill(reap=reap, accumulator=imatrix) + arrays = {**imatrix.as_arrays(), **reap.stats().as_arrays()} + counts = arrays["reap__route_count"].copy() + nonzero = np.argwhere(counts > 1)[0] + counts[tuple(nonzero)] -= 1 + counts[nonzero[0], (nonzero[1] + 1) % counts.shape[1]] += 1 + arrays["reap__route_count"] = counts + with pytest.raises(ValueError, match="REAP"): + validate_dsv4_calibration_capture( + arrays, layers=layers, num_experts=args.n_routed_experts + ) + + def test_calibration_run_writes_reap_stats_and_finalize_sums_them(self, tmp_path): + from keep.quality.dsv4_teacher_runner import finalize_dsv4_reap_stats + from keep.quality.reap import ReapStats + + args, model = _tiny_model() + experts = _tiny_experts(args) + pack = _write_tiny_pack(tmp_path / "pack.json", vocab=args.vocab_size) + out = tmp_path / "out" + run_dsv4_teacher_production( + mode="calibration", + pack_path=pack, + out_dir=out, + chunk=16, + workload_loader=_tiny_loader(args, model, experts), + ) + report = finalize_dsv4_reap_stats(out) + assert report["session_count"] == 2 + stats = ReapStats.load(report["output"]) + assert len(stats.sessions) == 2 + assert stats.routed_slots.sum() == report["routed_slots"] > 0 + + # A session without REAP arrays (e.g. produced before this existed) + # makes the finalizer refuse rather than under-count. + victim = sorted((out / "sessions").glob("*.npz"))[0] + with np.load(victim, allow_pickle=False) as archive: + kept = {k: archive[k] for k in archive.files if not k.startswith("reap__")} + np.savez(victim, **kept) + with pytest.raises(ValueError, match="no REAP statistics"): + finalize_dsv4_reap_stats(out) + + class TestStreamStats: def test_device_rate_uses_busy_time_not_the_overlapped_span(self): """The span is not a throughput; only ``io_busy_seconds`` is. diff --git a/tests/test_reap.py b/tests/test_reap.py new file mode 100644 index 0000000..36e26e5 --- /dev/null +++ b/tests/test_reap.py @@ -0,0 +1,333 @@ +"""REAP saliency from the real routed forward, survivor choice, and the prune plan.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from keep.quality.reap import ( + ReapStats, + expert_saliency, + prune_checkpoint_plan, + pruned_set_jaccard, + select_survivors, + super_expert_candidates, +) + + +def _stats(route_count, gate_norm_sum, *, abs_max=None, gate_sum=None, norm_sum=None): + route_count = np.asarray(route_count, dtype=np.int64) + gate_norm_sum = np.asarray(gate_norm_sum, dtype=np.float64) + shape = route_count.shape + return ReapStats( + layers=np.arange(shape[0]), + num_experts=shape[1], + route_count=route_count, + gate_sum=np.zeros(shape) if gate_sum is None else gate_sum, + gate_norm_sum=gate_norm_sum, + norm_sum=np.zeros(shape) if norm_sum is None else norm_sum, + norm_max=np.zeros(shape), + abs_max=np.zeros(shape) if abs_max is None else abs_max, + routed_slots=route_count.sum(axis=1), + ) + + +# -- saliency ----------------------------------------------------------------- + + +def test_reap_is_the_mean_gate_weighted_norm_over_routed_slots(): + stats = _stats([[4, 1, 0]], [[2.0, 3.0, 0.0]]) + np.testing.assert_allclose(expert_saliency(stats, "reap"), [[0.5, 3.0, 0.0]]) + # A rarely routed expert with large outputs outranks a frequent quiet one, + # which is exactly where REAP and frequency pruning disagree. + np.testing.assert_array_equal(expert_saliency(stats, "frequency"), [[4, 1, 0]]) + + +def test_never_routed_experts_score_zero_under_every_criterion(): + stats = _stats([[3, 0]], [[1.5, 0.0]], gate_sum=[[0.9, 0.0]], norm_sum=[[6.0, 0.0]]) + for criterion in ("reap", "frequency", "weighted_frequency", "ean_mean"): + assert expert_saliency(stats, criterion)[0, 1] == 0.0 + + +def test_prior_shrinks_rare_experts_toward_the_layer_mean(): + stats = _stats([[1000, 1]], [[1000.0, 50.0]]) # layer mean ~1.05 + raw = expert_saliency(stats, "reap") + shrunk = expert_saliency(stats, "reap", prior_slots=100.0) + assert raw[0, 1] == pytest.approx(50.0) + assert 1.0 < shrunk[0, 1] < 2.0 # one noisy slot no longer dominates + assert shrunk[0, 0] == pytest.approx(raw[0, 0], rel=0.01) + + +def test_unknown_criterion_and_negative_prior_are_refused(): + stats = _stats([[1]], [[1.0]]) + with pytest.raises(ValueError): + expert_saliency(stats, "softmax") + with pytest.raises(ValueError): + expert_saliency(stats, "reap", prior_slots=-1) + + +# -- survivors and comparisons --------------------------------------------------- + + +def test_survivors_are_the_most_salient_in_ascending_id_order(): + saliency = np.array([[0.1, 0.9, 0.5, 0.7], [1.0, 0.0, 0.0, 0.2]]) + np.testing.assert_array_equal(select_survivors(saliency, 2), [[1, 3], [0, 3]]) + + +def test_ties_break_toward_the_lower_expert_id(): + np.testing.assert_array_equal(select_survivors(np.array([[0.5, 0.5, 0.5]]), 2), [[0, 1]]) + + +@pytest.mark.parametrize("keep", [0, 5]) +def test_keep_must_fit_the_expert_count(keep): + with pytest.raises(ValueError): + select_survivors(np.zeros((1, 4)), keep) + + +def test_pruned_set_jaccard(): + a = np.array([[0, 1], [0, 1]]) + b = np.array([[0, 1], [2, 3]]) + np.testing.assert_allclose(pruned_set_jaccard(a, b, num_experts=4), [1.0, 0.0]) + + +def test_super_expert_screen_sees_negative_outliers_through_abs_max(): + # Enough experts that the 99.5th percentile describes the bulk, as it does + # at 40 x 384 (it would be the outliers themselves at 4 x 8). + abs_max = np.full((8, 64), 1.0) + abs_max[1, 5] = 900.0 # e.g. a large negative output, invisible to a signed max + abs_max[7, 2] = 950.0 # last quarter of layers is excluded, as upstream does + stats = _stats(np.ones((8, 64), dtype=np.int64), np.ones((8, 64)), abs_max=abs_max) + assert super_expert_candidates(stats) == [(1, 5)] + + +# -- stats container ---------------------------------------------------------------- + + +def test_stats_validate_their_invariants(): + with pytest.raises(ValueError, match="sum to the routed slots"): + ReapStats( + layers=[0], + num_experts=2, + route_count=[[1, 1]], + gate_sum=np.zeros((1, 2)), + gate_norm_sum=np.zeros((1, 2)), + norm_sum=np.zeros((1, 2)), + norm_max=np.zeros((1, 2)), + abs_max=np.zeros((1, 2)), + routed_slots=[3], + ) + with pytest.raises(ValueError, match="no routed slots"): + _stats([[0, 2]], [[1.0, 1.0]]) + with pytest.raises(ValueError, match="non-negative"): + _stats([[1, 1]], [[-1.0, 1.0]]) + + +def test_stats_round_trip_and_merge(tmp_path): + a = _stats([[2, 1]], [[1.0, 2.0]], abs_max=np.array([[3.0, 1.0]])) + a.sessions = ("s0",) + b = _stats([[1, 3]], [[0.5, 1.0]], abs_max=np.array([[1.0, 4.0]])) + b.sessions = ("s1",) + merged = a.merged(b) + np.testing.assert_array_equal(merged.route_count, [[3, 4]]) + np.testing.assert_allclose(merged.gate_norm_sum, [[1.5, 3.0]]) + np.testing.assert_allclose(merged.abs_max, [[3.0, 4.0]]) + assert merged.sessions == ("s0", "s1") + with pytest.raises(ValueError, match="double-count"): + merged.merged(a) + + path = tmp_path / "reap-stats.npz" + merged.save(path) + loaded = ReapStats.load(path) + assert loaded.sessions == ("s0", "s1") + np.testing.assert_array_equal(loaded.route_count, merged.route_count) + np.testing.assert_allclose(loaded.gate_norm_sum, merged.gate_norm_sum) + + +# -- device accumulator --------------------------------------------------------------- + + +@pytest.mark.mlx +def test_accumulator_matches_a_float64_reference(): + mx = pytest.importorskip("mlx.core") + from keep.quality.reap import ReapAccumulator + + rng = np.random.default_rng(3) + experts, top_k, hidden = 6, 3, 16 + acc = ReapAccumulator(layers=[4, 9], num_experts=experts) + ref = {name: np.zeros((2, experts)) for name in ("count", "g", "gn", "n", "nmax", "amax")} + for layer_row, layer in enumerate((4, 9)): + for rows in (5, 11): # two chunks per layer + ids = rng.integers(0, experts, size=(rows, top_k)) + gates = rng.random((rows, top_k)).astype(np.float32) + outs = rng.normal(size=(rows, top_k, hidden)).astype(np.float32) * 3 + acc.add_chunk( + layer=layer, + expert_ids=mx.array(ids), + gate_weights=mx.array(gates), + expert_outputs=mx.array(outs), + ) + acc.eval_pending() + norms = np.linalg.norm(outs.astype(np.float64), axis=-1) + peaks = np.abs(outs).max(axis=-1) + for e, g, n, p in zip(ids.ravel(), gates.ravel(), norms.ravel(), peaks.ravel()): + ref["count"][layer_row, e] += 1 + ref["g"][layer_row, e] += g + ref["gn"][layer_row, e] += g * n + ref["n"][layer_row, e] += n + ref["nmax"][layer_row, e] = max(ref["nmax"][layer_row, e], n) + ref["amax"][layer_row, e] = max(ref["amax"][layer_row, e], p) + acc.flush_device() + stats = acc.stats(session="sess") + np.testing.assert_array_equal(stats.route_count, ref["count"]) + np.testing.assert_allclose(stats.gate_sum, ref["g"], rtol=2e-6) + np.testing.assert_allclose(stats.gate_norm_sum, ref["gn"], rtol=2e-6) + np.testing.assert_allclose(stats.norm_sum, ref["n"], rtol=2e-6) + np.testing.assert_allclose(stats.norm_max, ref["nmax"], rtol=2e-6) + np.testing.assert_allclose(stats.abs_max, ref["amax"], rtol=1e-7) + assert stats.sessions == ("sess",) + assert stats.routed_slots.tolist() == [48, 48] + + +@pytest.mark.mlx +def test_accumulator_refuses_mismatched_shapes_and_untracked_layers(): + mx = pytest.importorskip("mlx.core") + from keep.quality.reap import ReapAccumulator + + acc = ReapAccumulator(layers=[0], num_experts=4) + ids = mx.zeros((2, 2), dtype=mx.int32) + with pytest.raises(KeyError): + acc.add_chunk(layer=1, expert_ids=ids, gate_weights=ids, expert_outputs=mx.zeros((2, 2, 3))) + with pytest.raises(ValueError, match="do not match"): + acc.add_chunk(layer=0, expert_ids=ids, gate_weights=ids, expert_outputs=mx.zeros((2, 3, 3))) + + +# -- slicing the router is the same as masking it --------------------------------------- + + +@pytest.mark.mlx +def test_sliced_gate_equals_the_full_gate_restricted_to_survivors(): + """The prune applies by slicing router rows; check it against masking. + + Masking pruned experts out of *selection* (correction bias -> -inf) is + what "drop the expert and re-select top-k among survivors" means. The + sliced gate must pick the same experts with the same renormalised + weights, mapped back through the survivor table. + """ + + mx = pytest.importorskip("mlx.core") + from ramp.models.deepseek_v4_flash_adapter import DeepseekV4FlashVQModelArgs, MoEGate + + rng = np.random.default_rng(11) + experts, keep, hidden = 12, 7, 32 + args = DeepseekV4FlashVQModelArgs( + hidden_size=hidden, + n_routed_experts=experts, + num_experts_per_tok=3, + num_hash_layers=0, + num_hidden_layers=1, + compress_ratios=[0], + ) + weight = rng.normal(size=(experts, hidden)).astype(np.float32) + bias = rng.normal(size=(experts,)).astype(np.float32) * 0.1 + survivors = np.sort(rng.choice(experts, size=keep, replace=False)) + + full = MoEGate(args, 0) + full.weight = mx.array(weight) + masked_bias = bias.copy() + masked_bias[np.setdiff1d(np.arange(experts), survivors)] = -1e30 + full.e_score_correction_bias = mx.array(masked_bias) + + sliced_args = DeepseekV4FlashVQModelArgs( + hidden_size=hidden, + n_routed_experts=keep, + num_experts_per_tok=3, + num_hash_layers=0, + num_hidden_layers=1, + compress_ratios=[0], + ) + sliced = MoEGate(sliced_args, 0) + sliced.weight = mx.array(weight[survivors]) + sliced.e_score_correction_bias = mx.array(bias[survivors]) + + x = mx.array(rng.normal(size=(1, 64, hidden)).astype(np.float32)) + full_ids, full_scores = full(x) + sliced_ids, sliced_scores = sliced(x) + mapped = survivors[np.array(sliced_ids)] + np.testing.assert_array_equal(np.sort(mapped, axis=-1), np.sort(np.array(full_ids), axis=-1)) + order_full = np.argsort(np.array(full_ids), axis=-1) + order_sliced = np.argsort(mapped, axis=-1) + np.testing.assert_allclose( + np.take_along_axis(np.array(sliced_scores), order_sliced, axis=-1), + np.take_along_axis(np.array(full_scores), order_full, axis=-1), + rtol=1e-6, + ) + + +# -- checkpoint prune plan ------------------------------------------------------------ + + +def _names(layers=2, experts=4, *, hash_layer=None, with_bias_vl=True): + names = ["embed.weight", "head.weight", "mtp.0.ffn.gate.weight", "mtp.0.ffn.experts.0.w1.weight"] + for layer in range(layers): + p = f"layers.{layer}" + names += [f"{p}.attn.wq_a.weight", f"{p}.ffn.gate.weight", f"{p}.ffn.shared_experts.w1.weight"] + if layer == hash_layer: + names.append(f"{p}.ffn.gate.tid2eid") + else: + names.append(f"{p}.ffn.gate.bias") + if with_bias_vl: + names.append(f"{p}.ffn.gate.bias_vl") + for e in range(experts): + for proj in ("w1", "w2", "w3"): + names += [f"{p}.ffn.experts.{e}.{proj}.weight", f"{p}.ffn.experts.{e}.{proj}.scale"] + return names + + +def test_prune_plan_renumbers_slices_and_drops(): + plan = prune_checkpoint_plan(_names(), {0: [1, 3], 1: [0, 2]}) + assert plan["keep"] == 2 + assert plan["rename"]["layers.0.ffn.experts.3.w2.scale"] == "layers.0.ffn.experts.1.w2.scale" + assert plan["rename"]["layers.1.ffn.experts.0.w1.weight"] == "layers.1.ffn.experts.0.w1.weight" + assert "layers.0.ffn.experts.0.w1.weight" in plan["drop"] + assert len(plan["rename"]) == 2 * 2 * 3 * 2 + assert len(plan["drop"]) == 2 * 2 * 3 * 2 + # bias_vl is sliced with the gate, never left at the old width. + for leaf in ("weight", "bias", "bias_vl"): + assert plan["slice_rows"][f"layers.0.ffn.gate.{leaf}"] == [1, 3] + # Attention, shared experts, embeddings and the drafter are untouched. + for name in ("layers.0.attn.wq_a.weight", "layers.1.ffn.shared_experts.w1.weight", + "embed.weight", "mtp.0.ffn.gate.weight", "mtp.0.ffn.experts.0.w1.weight"): + assert name in plan["untouched"] + + +def test_prune_plan_refuses_hash_layers_and_ragged_or_phantom_survivors(): + with pytest.raises(NotImplementedError, match="hash-routed"): + prune_checkpoint_plan(_names(hash_layer=0), {0: [0, 1], 1: [0, 1]}) + with pytest.raises(ValueError, match="same expert count"): + prune_checkpoint_plan(_names(), {0: [0, 1], 1: [0, 1, 2]}) + with pytest.raises(ValueError, match="no tensors"): + prune_checkpoint_plan(_names(experts=4), {0: [0, 9], 1: [0, 1]}) + with pytest.raises(ValueError, match="duplicate"): + prune_checkpoint_plan(_names(), {0: [1, 1], 1: [0, 1]}) + + +def test_reap_select_command_reports_the_kill_test_and_writes_survivors(tmp_path, capsys): + import json + + from benchmarks import produce_dsv4_teacher_cache as cli + + rng = np.random.default_rng(2) + counts = rng.integers(0, 50, size=(3, 16)) + stats = _stats(counts, counts * rng.random((3, 16)) * 2) + stats.sessions = ("a", "b") + stats_path = tmp_path / "reap-stats.npz" + stats.save(stats_path) + out = tmp_path / "survivors.json" + + assert cli.main(["reap-select", "--stats", str(stats_path), "--keep", "8", "--output", str(out)]) == 0 + report = json.loads(capsys.readouterr().out) + assert report["keep"] == 8 and report["compare"] == "frequency" + assert set(report["pruned_set_jaccard_vs_compare"]) == {"0", "1", "2"} + saved = json.loads(out.read_text()) + assert saved["record_type"] == "keep_reap_survivors_v1" + assert all(len(ids) == 8 and ids == sorted(ids) for ids in saved["survivors"].values()) From 33a5681df826cc7bb66e95f1a6796863b0bfad40 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:15:05 +0000 Subject: [PATCH 04/74] keep: replay recorded routing through an in-RAM expert hot set (SSD-tier kill test) Keeping every routed expert of a model too large for RAM means a hot set in memory and the rest read from SSD on a miss, on the decode critical path. keep.quality.expert_residency answers whether that is fast enough before an offload engine is built: - RouteRecorder keeps the first 2,048 tokens of routing per layer. In a causal model those prefill routes are also the decode routes. DSV4 calibration sessions now record them (routes__expert_ids int16, routes__gate_weights float16, about 1 MB per session). - simulate_residency replays them through a per-layer hot set under three policies: static (a fixed REAP-ordered set), LRU, and decayed use counts with a half-life (oMLX's V4.1 policy). It can warm-start from REAP saliency. fetch_threshold models a hybrid that drops light misses as a pruned model would and reports the gate mass it drops. simulate_sessions resets residency at each session boundary and pools raw counts. - residency_report applies the plan's rule per hot-set size: KILL if the best policy that fetches every miss still misses more than 5% of routed slots. decode_tokens_per_second turns misses into a tok/s estimate for a given SSD rate. - benchmarks/replay_dsv4_expert_residency.py runs it over a calibration run's sessions plus reap-stats.npz. layer_major_prefill's new hook is generalised to route_observers, so the REAP accumulator and the route recorder ride the same pass. The forward is still bit-identical. --- benchmarks/replay_dsv4_expert_residency.py | 153 ++++++++++ src/keep/quality/dsv4_teacher_runner.py | 43 ++- src/keep/quality/expert_residency.py | 339 +++++++++++++++++++++ tests/test_dsv4_teacher_runner.py | 11 +- tests/test_expert_residency.py | 206 +++++++++++++ 5 files changed, 736 insertions(+), 16 deletions(-) create mode 100644 benchmarks/replay_dsv4_expert_residency.py create mode 100644 src/keep/quality/expert_residency.py create mode 100644 tests/test_expert_residency.py diff --git a/benchmarks/replay_dsv4_expert_residency.py b/benchmarks/replay_dsv4_expert_residency.py new file mode 100644 index 0000000..f62ee60 --- /dev/null +++ b/benchmarks/replay_dsv4_expert_residency.py @@ -0,0 +1,153 @@ +"""Kill test: replay recorded routing through an in-RAM hot set of experts. + +Reads the ``routes__*`` arrays that DSV4 calibration sessions record (the +first tokens of each session, which are also its decode routing) and, if +given, ``reap-stats.npz`` to choose the initial hot set in REAP order. Replays +every hot-set size and policy asked for and applies the plan's rule: at a +given hot-set size, if the best policy that fetches every miss still misses +more than 5% of routed slots, keeping all experts behind an SSD tier at that +size is too slow to be worth building. + +Example (V4-Flash, half the experts resident):: + + uv run python benchmarks/replay_dsv4_expert_residency.py \\ + --calibration-dir ~/keep-artifacts/dsv4-teacher-calibration \\ + --reap-stats ~/keep-artifacts/dsv4-teacher-calibration/reap-stats.npz \\ + --hot 96,128,160 --expert-mb 6.39 --output residency.json +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +import numpy as np + +from keep.quality.expert_residency import ( + ROUTE_ARRAY_PREFIX, + decode_tokens_per_second, + residency_report, + simulate_sessions, +) + + +def load_route_sessions(calibration_dir: Path) -> tuple[list[tuple[np.ndarray, np.ndarray]], np.ndarray]: + sessions: list[tuple[np.ndarray, np.ndarray]] = [] + layers: np.ndarray | None = None + missing: list[str] = [] + for path in sorted((calibration_dir / "sessions").glob("*.npz")): + with np.load(path, allow_pickle=False) as archive: + if f"{ROUTE_ARRAY_PREFIX}expert_ids" not in archive.files: + missing.append(path.name) + continue + current = archive[f"{ROUTE_ARRAY_PREFIX}layers"].astype(np.int64) + if layers is None: + layers = current + elif not np.array_equal(layers, current): + raise ValueError(f"{path.name} records a different layer set") + sessions.append( + ( + archive[f"{ROUTE_ARRAY_PREFIX}expert_ids"].astype(np.int64), + archive[f"{ROUTE_ARRAY_PREFIX}gate_weights"].astype(np.float32), + ) + ) + if missing: + raise ValueError(f"{len(missing)} sessions have no recorded routes: {missing[:5]}") + if not sessions or layers is None: + raise ValueError(f"no session routes under {calibration_dir / 'sessions'}") + return sessions, layers + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--calibration-dir", type=Path, required=True) + parser.add_argument("--reap-stats", type=Path, help="initial hot set in REAP order") + parser.add_argument("--num-experts", type=int, help="default: from --reap-stats") + parser.add_argument("--hot", default="128", help="comma-separated hot experts per layer") + parser.add_argument("--policies", default="static,lru,decayed") + parser.add_argument("--half-life", type=float, default=64.0) + parser.add_argument("--fetch-threshold", type=float, help="drop misses below this gate weight") + parser.add_argument("--warmup-tokens", type=int, default=0) + parser.add_argument("--expert-mb", type=float, default=8.9856, help="bytes per expert, MB (V4.1 at 2.031 bpw)") + parser.add_argument("--resident-ms", type=float, help="resident-only decode ms/token, for a tok/s estimate") + parser.add_argument("--ssd-gb-s", type=float, default=5.0) + parser.add_argument("--kill-miss-rate", type=float, default=0.05) + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + + sessions, layers = load_route_sessions(args.calibration_dir.expanduser()) + initial = None + num_experts = args.num_experts + if args.reap_stats is not None: + from keep.quality.reap import ReapStats, expert_saliency + + stats = ReapStats.load(args.reap_stats.expanduser()) + if not np.array_equal(stats.layers, layers): + parser.error("--reap-stats covers different layers than the recorded routes") + initial = expert_saliency(stats, "reap") + num_experts = num_experts or stats.num_experts + if num_experts is None: + parser.error("pass --num-experts or --reap-stats") + policies = [p.strip() for p in args.policies.split(",") if p.strip()] + if "static" in policies and initial is None: + policies.remove("static") # a static set needs an order to choose it + expert_bytes = args.expert_mb * 1e6 + results = [] + for hot in (int(h) for h in args.hot.split(",")): + for policy in policies: + results.append( + simulate_sessions( + sessions, + num_experts=num_experts, + hot_per_layer=hot, + policy=policy, + half_life=args.half_life, + initial_order=initial, + expert_bytes=expert_bytes, + warmup_tokens=args.warmup_tokens, + ) + ) + if args.fetch_threshold is not None: + results.append( + simulate_sessions( + sessions, + num_experts=num_experts, + hot_per_layer=hot, + policy=policy, + half_life=args.half_life, + initial_order=initial, + expert_bytes=expert_bytes, + warmup_tokens=args.warmup_tokens, + fetch_threshold=args.fetch_threshold, + ) + ) + report = dict(residency_report(results, kill_miss_rate=args.kill_miss_rate)) + report["sessions"] = len(sessions) + report["layers"] = layers.tolist() + report["expert_mb"] = args.expert_mb + if args.resident_ms is not None: + report["tokens_per_second_estimate"] = { + f"{r.policy}@{r.hot_per_layer}" + + ("" if r.dropped == 0 else f"+drop<{args.fetch_threshold}"): round( + decode_tokens_per_second( + resident_ms_per_token=args.resident_ms, + misses_per_token=r.misses_per_token, + expert_bytes=expert_bytes, + ssd_bytes_per_second=args.ssd_gb_s * 1e9, + ), + 2, + ) + for r in results + } + text = json.dumps(report, indent=2, sort_keys=True) + if args.output is not None: + args.output.write_text(text + "\n") + print(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/keep/quality/dsv4_teacher_runner.py b/src/keep/quality/dsv4_teacher_runner.py index aa12438..6ebdc8c 100644 --- a/src/keep/quality/dsv4_teacher_runner.py +++ b/src/keep/quality/dsv4_teacher_runner.py @@ -1163,7 +1163,7 @@ def layer_major_prefill( on_layer: Callable[[int, int], None] | None = None, release_after_layer: bool = True, dspark_taps: dict[int, list[Any]] | None = None, - reap: Any | None = None, + route_observers: Sequence[Any] = (), ) -> list[Any]: """Prefill the whole backbone layer-outer, chunk-inner. @@ -1185,11 +1185,14 @@ def layer_major_prefill( to be a dict of lists rather than one concatenated array: the taps for layer 40 are complete before layer 41 has run at all. - ``reap`` is an optional :class:`keep.quality.reap.ReapAccumulator`. It - takes the model's own gate weights and each routed expert's output before - gating, so it scores experts against the routing the model actually uses. - Like the imatrix it reads tensors the forward already computes and leaves - the forward bit-identical. + ``route_observers`` receive every chunk's routing as the model computed + it: ``add_chunk(layer=, expert_ids=, gate_weights=, expert_outputs=)`` + with the model's own gate weights and each routed expert's output before + gating, then ``eval_pending()`` per chunk and ``flush_device()`` per + layer. :class:`keep.quality.reap.ReapAccumulator` and + :class:`keep.quality.expert_residency.RouteRecorder` are the two users. + Like the imatrix, observers read tensors the forward already computes and + leave it bit-identical. """ import mlx.core as mx @@ -1292,7 +1295,7 @@ def down_hook(activated, expert_ids, order, _capture=capture): ) mask = masks[chunk_index] - if accumulator is None and reap is None: + if accumulator is None and not route_observers: hidden[chunk_index] = layer(h, mask, layer_cache, piece) if layer_index in tap_layers: tap = _dspark_tap(hidden[chunk_index]) @@ -1324,8 +1327,8 @@ def down_hook(activated, expert_ids, order, _capture=capture): flat_inds = inds.reshape(-1, inds.shape[-1]) flat_scores = scores.reshape(-1, scores.shape[-1]).astype(mx.float32) - if reap is not None: - reap.add_chunk( + for observer in route_observers: + observer.add_chunk( layer=layer_index, expert_ids=flat_inds, gate_weights=flat_scores, @@ -1333,7 +1336,8 @@ def down_hook(activated, expert_ids, order, _capture=capture): ) if accumulator is None: mx.eval(hidden[chunk_index]) - reap.eval_pending() + for observer in route_observers: + observer.eval_pending() continue if collect_hidden: rows = moe_input.reshape(-1, moe_input.shape[-1]).astype(mx.float32) @@ -1363,8 +1367,8 @@ def down_hook(activated, expert_ids, order, _capture=capture): ) mx.eval(hidden[chunk_index]) accumulator.eval_pending() - if reap is not None: - reap.eval_pending() + for observer in route_observers: + observer.eval_pending() switch_mlp.down_hook = None layer.ffn.switch_mlp = None @@ -1373,8 +1377,8 @@ def down_hook(activated, expert_ids, order, _capture=capture): expert_provider.release(layer_index) if accumulator is not None: accumulator.flush_device() - if reap is not None: - reap.flush_device() + for observer in route_observers: + observer.flush_device() mx.clear_cache() if on_layer is not None: on_layer(layer_index, n_chunks) @@ -2702,11 +2706,19 @@ def _produce_sessions( token_ids = mx.array(session.token_ids.astype(np.int64)[None, :]) accumulator = None reap = None + routes = None + observers: tuple[Any, ...] = () collect: tuple[str, ...] = () if mode == "calibration": + from keep.quality.expert_residency import ( + ROUTE_RECORD_TOKENS, + RouteRecorder, + ) from keep.quality.reap import ReapAccumulator reap = ReapAccumulator(layers=layers, num_experts=num_experts) + routes = RouteRecorder(layers=layers, max_tokens=ROUTE_RECORD_TOKENS) + observers = (reap, routes) accumulator = _ImatrixAccumulator( layers=layers, num_experts=num_experts, @@ -2736,13 +2748,14 @@ def on_layer(layer_index: int, n_chunks: int, _s=session, _i=index) -> None: collect_spaces=collect, on_layer=on_layer, dspark_taps=taps, - reap=reap, + route_observers=observers, ) if mode == "calibration": arrays = { **calibration_session_arrays(accumulator), **reap.stats(session=session.prompt_id).as_arrays(), + **routes.as_arrays(), } validate_dsv4_calibration_capture( arrays, layers=layers, num_experts=num_experts diff --git a/src/keep/quality/expert_residency.py b/src/keep/quality/expert_residency.py new file mode 100644 index 0000000..bd66fd1 --- /dev/null +++ b/src/keep/quality/expert_residency.py @@ -0,0 +1,339 @@ +"""Would a hot set of experts in RAM, with the rest on SSD, keep decode fast? + +The kill test for keeping every routed expert of a model too large for RAM: +replay real routing through a fixed number of resident experts per layer and +count how often a selected expert is not resident. Each miss is an SSD read of +one expert (about 9 MB at 2.031 bpw on DeepSeek-V4.1) on the decode critical +path, so the miss rate converts directly into bytes and milliseconds. + +Routing is a property of the token, not of how it was computed: in a causal +model the experts token ``t`` selects in a prefill pass are the ones it selects +when decoded. So routes recorded during the teacher's calibration prefill +(:class:`RouteRecorder`) are the decode routing sequence for those tokens. + +Policies replayed by :func:`simulate_residency`: + +- ``static``: the resident set never changes (the REAP-ordered hot set a + pruned-plus-SSD artifact would ship). Misses are streamed, not cached. +- ``lru``: evict the least recently used resident. +- ``decayed``: evict the resident with the lowest exponentially decayed use + count (half-life in tokens), the policy oMLX's DeepSeek-V4.1 offload uses. + +``fetch_threshold`` models the hybrid "soft prune": a miss whose gate weight +is below the threshold is dropped (the expert is skipped and the remaining +weights renormalised, exactly as a pruned model would), and only heavier +misses are fetched. The result reports how much gate mass that drops. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from typing import Any, Literal + +import numpy as np + +__all__ = [ + "ROUTE_ARRAY_PREFIX", + "ROUTE_RECORD_TOKENS", + "ResidencyResult", + "RouteRecorder", + "decode_tokens_per_second", + "residency_report", + "simulate_residency", + "simulate_sessions", +] + +ROUTE_ARRAY_PREFIX = "routes__" +#: Tokens of routing recorded per calibration session: enough for a decode- +#: length replay, about 1 MB per session at 43 layers x top-6. +ROUTE_RECORD_TOKENS = 2048 +Policy = Literal["static", "lru", "decayed"] + + +class RouteRecorder: + """Keep the first ``max_tokens`` routes of every layer, in token order. + + Implements the teacher runner's route-observer interface. The layer loop + is layer-outer and chunk-inner, so each layer's rows arrive in token order. + """ + + def __init__(self, *, layers: Sequence[int], max_tokens: int) -> None: + if max_tokens <= 0: + raise ValueError("max_tokens must be positive") + self.layers = tuple(int(layer) for layer in layers) + self.max_tokens = int(max_tokens) + self._ids: dict[int, list[np.ndarray]] = {layer: [] for layer in self.layers} + self._gates: dict[int, list[np.ndarray]] = {layer: [] for layer in self.layers} + self._count = {layer: 0 for layer in self.layers} + + def add_chunk( + self, *, layer: int, expert_ids: Any, gate_weights: Any, expert_outputs: Any = None + ) -> None: + del expert_outputs # routing only + if layer not in self._count: + raise KeyError(f"layer {layer} is not tracked by this recorder") + room = self.max_tokens - self._count[layer] + if room <= 0: + return + ids = np.asarray(expert_ids)[:room] + gates = np.asarray(gate_weights, dtype=np.float32)[:room] + if ids.shape != gates.shape or ids.ndim != 2: + raise ValueError("expert ids and gate weights must share a [rows, top_k] shape") + self._ids[layer].append(ids.astype(np.int16)) + self._gates[layer].append(gates.astype(np.float16)) + self._count[layer] += int(ids.shape[0]) + + def eval_pending(self) -> None: + return None + + def flush_device(self) -> None: + return None + + def as_arrays(self) -> dict[str, np.ndarray]: + counts = {self._count[layer] for layer in self.layers} + if len(counts) != 1: + raise ValueError(f"layers recorded different token counts: {sorted(counts)}") + ids = np.stack([np.concatenate(self._ids[layer]) for layer in self.layers], axis=1) + gates = np.stack([np.concatenate(self._gates[layer]) for layer in self.layers], axis=1) + return { + f"{ROUTE_ARRAY_PREFIX}layers": np.asarray(self.layers, dtype=np.int64), + f"{ROUTE_ARRAY_PREFIX}expert_ids": ids, # int16 [tokens, layers, top_k] + f"{ROUTE_ARRAY_PREFIX}gate_weights": gates, # float16, same shape + } + + +@dataclass(frozen=True) +class ResidencyResult: + policy: str + hot_per_layer: int + num_experts: int + tokens: int + routed_slots: int + misses: int # fetched from SSD + dropped: int # skipped under fetch_threshold + miss_rate: float + misses_per_token: float + dropped_gate_mass_fraction: float + ssd_bytes_per_token: float | None + per_layer_miss_rate: tuple[float, ...] + dropped_gate_mass: float = 0.0 + total_gate_mass: float = 0.0 + per_layer_misses: tuple[int, ...] = () + + def to_dict(self) -> dict[str, Any]: + out = asdict(self) + out["per_layer_miss_rate"] = list(self.per_layer_miss_rate) + return out + + +def simulate_residency( + expert_ids: np.ndarray, + *, + num_experts: int, + hot_per_layer: int, + policy: Policy = "decayed", + half_life: float = 64.0, + initial_order: np.ndarray | None = None, + gate_weights: np.ndarray | None = None, + fetch_threshold: float | None = None, + expert_bytes: float | None = None, + warmup_tokens: int = 0, +) -> ResidencyResult: + """Replay ``expert_ids [tokens, layers, top_k]`` through a per-layer hot set. + + ``initial_order`` is ``[layers, experts]`` saliency (e.g. REAP scores); the + hot set starts as its top ``hot_per_layer`` experts. Without it the set + starts empty and fills on demand. ``warmup_tokens`` are replayed but not + counted. + """ + + ids = np.asarray(expert_ids) + if ids.ndim != 3: + raise ValueError("expert_ids must be [tokens, layers, top_k]") + tokens, layers, top_k = ids.shape + if not 0 < hot_per_layer <= num_experts: + raise ValueError(f"hot_per_layer must be in 1..{num_experts}") + if hot_per_layer < top_k: + raise ValueError("the hot set must hold at least one token's top-k") + if np.any(ids < 0) or np.any(ids >= num_experts): + raise ValueError("expert ids outside the expert range") + if policy not in ("static", "lru", "decayed"): + raise ValueError(f"unknown policy {policy!r}") + if policy == "static" and initial_order is None: + raise ValueError("a static hot set needs initial_order to choose it") + if fetch_threshold is not None and gate_weights is None: + raise ValueError("fetch_threshold needs gate_weights") + gates = None if gate_weights is None else np.asarray(gate_weights, dtype=np.float64) + if gates is not None and gates.shape != ids.shape: + raise ValueError("gate_weights must match expert_ids") + if not 0 <= warmup_tokens < tokens: + raise ValueError("warmup_tokens must leave at least one counted token") + decay = np.log(2.0) / float(half_life) + + misses = np.zeros(layers, dtype=np.int64) + dropped = 0 + dropped_mass = 0.0 + total_mass = 0.0 + for layer in range(layers): + resident = np.zeros(num_experts, dtype=bool) + if initial_order is not None: + order = np.lexsort((np.arange(num_experts), -np.asarray(initial_order)[layer])) + resident[order[:hot_per_layer]] = True + score = np.zeros(num_experts, dtype=np.float64) # decayed count at `stamp` + stamp = np.zeros(num_experts, dtype=np.float64) + last_use = np.full(num_experts, -1.0) + for t in range(tokens): + counted = t >= warmup_tokens + needed = ids[t, layer] + needed_set = set(int(e) for e in needed) + for slot, expert in enumerate(needed): + expert = int(expert) + weight = None if gates is None else float(gates[t, layer, slot]) + if counted and weight is not None: + total_mass += weight + if not resident[expert]: + if fetch_threshold is not None and weight < fetch_threshold: + if counted: + dropped += 1 + dropped_mass += weight + continue + if counted: + misses[layer] += 1 + if policy != "static": + if resident.sum() >= hot_per_layer: + candidates = np.flatnonzero(resident) + candidates = candidates[ + [c not in needed_set for c in candidates.tolist()] + ] + if policy == "lru": + victim = candidates[np.argmin(last_use[candidates])] + else: + current = score[candidates] * np.exp(-decay * (t - stamp[candidates])) + victim = candidates[np.argmin(current)] + resident[victim] = False + resident[expert] = True + # Usage bookkeeping for every selected expert, resident or not. + score[expert] = score[expert] * np.exp(-decay * (t - stamp[expert])) + 1.0 + stamp[expert] = t + last_use[expert] = t + counted_tokens = tokens - warmup_tokens + slots = counted_tokens * layers * top_k + total_misses = int(misses.sum()) + per_layer = tuple(float(m) / (counted_tokens * top_k) for m in misses) + return ResidencyResult( + policy=policy, + hot_per_layer=int(hot_per_layer), + num_experts=int(num_experts), + tokens=int(counted_tokens), + routed_slots=int(slots), + misses=total_misses, + dropped=int(dropped), + miss_rate=total_misses / slots, + misses_per_token=total_misses / counted_tokens, + dropped_gate_mass_fraction=(dropped_mass / total_mass) if total_mass else 0.0, + ssd_bytes_per_token=None if expert_bytes is None else total_misses / counted_tokens * float(expert_bytes), + per_layer_miss_rate=per_layer, + dropped_gate_mass=float(dropped_mass), + total_gate_mass=float(total_mass), + per_layer_misses=tuple(int(m) for m in misses), + ) + + +def simulate_sessions( + sessions: Sequence[tuple[np.ndarray, np.ndarray | None]], + *, + expert_bytes: float | None = None, + **kwargs: Any, +) -> ResidencyResult: + """Replay each session from the initial hot set and pool the counts. + + Residency is reset at every session boundary: unrelated sessions do not + share a warm cache in a real deployment's worst case, and pooling raw + counts weights each session by its tokens. + """ + + if not sessions: + raise ValueError("no sessions to replay") + parts = [ + simulate_residency(ids, gate_weights=gates, expert_bytes=expert_bytes, **kwargs) + for ids, gates in sessions + ] + tokens = sum(r.tokens for r in parts) + slots = sum(r.routed_slots for r in parts) + misses = sum(r.misses for r in parts) + layer_misses = np.sum([r.per_layer_misses for r in parts], axis=0) + top_k = slots // (tokens * len(layer_misses)) + dropped_mass = sum(r.dropped_gate_mass for r in parts) + total_mass = sum(r.total_gate_mass for r in parts) + first = parts[0] + return ResidencyResult( + policy=first.policy, + hot_per_layer=first.hot_per_layer, + num_experts=first.num_experts, + tokens=tokens, + routed_slots=slots, + misses=misses, + dropped=sum(r.dropped for r in parts), + miss_rate=misses / slots, + misses_per_token=misses / tokens, + dropped_gate_mass_fraction=(dropped_mass / total_mass) if total_mass else 0.0, + ssd_bytes_per_token=None if expert_bytes is None else misses / tokens * float(expert_bytes), + per_layer_miss_rate=tuple(float(m) / (tokens * top_k) for m in layer_misses), + dropped_gate_mass=dropped_mass, + total_gate_mass=total_mass, + per_layer_misses=tuple(int(m) for m in layer_misses), + ) + + +def decode_tokens_per_second( + *, + resident_ms_per_token: float, + misses_per_token: float, + expert_bytes: float, + ssd_bytes_per_second: float, + host_sync_ms_per_token: float = 0.0, +) -> float: + """Decode rate once misses are read synchronously on the critical path. + + ``host_sync_ms_per_token`` is the cost of learning which experts a layer + picked before issuing the reads (oMLX measured it as a large share of its + decode cycle); it is paid only by policies that fetch. + """ + + miss_ms = misses_per_token * expert_bytes / ssd_bytes_per_second * 1e3 + return 1e3 / (resident_ms_per_token + miss_ms + host_sync_ms_per_token) + + +def residency_report( + results: Sequence[ResidencyResult], *, kill_miss_rate: float = 0.05 +) -> Mapping[str, Any]: + """Summarise replays and apply the plan's kill rule per hot-set size. + + For each hot-set size, the best policy that *fetches* every miss (no + dropped gate mass) decides: above ``kill_miss_rate`` misses the SSD tier + costs too much at that size. + """ + + by_hot: dict[int, list[ResidencyResult]] = {} + for result in results: + by_hot.setdefault(result.hot_per_layer, []).append(result) + verdicts: dict[str, Any] = {} + for hot, group in sorted(by_hot.items()): + exact = [r for r in group if r.dropped == 0] + if not exact: + continue + best = min(exact, key=lambda r: r.miss_rate) + verdicts[str(hot)] = { + "best_policy": best.policy, + "miss_rate": best.miss_rate, + "misses_per_token": best.misses_per_token, + "ssd_bytes_per_token": best.ssd_bytes_per_token, + "verdict": "KILL" if best.miss_rate > kill_miss_rate else "PASS", + } + return { + "record_type": "keep_expert_residency_replay_v1", + "kill_miss_rate": kill_miss_rate, + "results": [result.to_dict() for result in results], + "by_hot_per_layer": verdicts, + } diff --git a/tests/test_dsv4_teacher_runner.py b/tests/test_dsv4_teacher_runner.py index f403fc8..664d8f2 100644 --- a/tests/test_dsv4_teacher_runner.py +++ b/tests/test_dsv4_teacher_runner.py @@ -1796,7 +1796,7 @@ def _prefill(self, *, reap=None, accumulator=None): accumulator=accumulator, collect_spaces=IMATRIX_SPACES if accumulator is not None else (), release_after_layer=False, - reap=reap, + route_observers=() if reap is None else (reap,), ) return args, out @@ -1888,6 +1888,15 @@ def test_calibration_run_writes_reap_stats_and_finalize_sums_them(self, tmp_path assert len(stats.sessions) == 2 assert stats.routed_slots.sum() == report["routed_slots"] > 0 + # The same pass records each session's routing for the residency replay. + first = sorted((out / "sessions").glob("*.npz"))[0] + with np.load(first, allow_pickle=False) as archive: + route_ids = archive["routes__expert_ids"] + assert route_ids.dtype == np.int16 + assert route_ids.shape[1:] == (args.num_hidden_layers, args.num_experts_per_tok) + assert route_ids.shape[0] > 0 + assert archive["routes__gate_weights"].shape == route_ids.shape + # A session without REAP arrays (e.g. produced before this existed) # makes the finalizer refuse rather than under-count. victim = sorted((out / "sessions").glob("*.npz"))[0] diff --git a/tests/test_expert_residency.py b/tests/test_expert_residency.py new file mode 100644 index 0000000..abd9611 --- /dev/null +++ b/tests/test_expert_residency.py @@ -0,0 +1,206 @@ +"""Replay of routing through an in-RAM hot set: the SSD-tier kill test.""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + +from keep.quality.expert_residency import ( + RouteRecorder, + decode_tokens_per_second, + residency_report, + simulate_residency, + simulate_sessions, +) + + +def _routes(tokens, layers, top_k, experts, seed=0, skew=1.2): + """Zipf-ish routing so a hot set is meaningful.""" + + rng = np.random.default_rng(seed) + weights = 1.0 / np.arange(1, experts + 1) ** skew + weights /= weights.sum() + ids = np.empty((tokens, layers, top_k), dtype=np.int64) + for t in range(tokens): + for layer in range(layers): + ids[t, layer] = rng.choice(experts, size=top_k, replace=False, p=weights) + gates = rng.dirichlet(np.ones(top_k), size=(tokens, layers)) + return ids, gates + + +def test_everything_resident_never_misses_once_warm(): + ids, _ = _routes(50, 2, 3, 8) + order = np.zeros((2, 8)) + warm = simulate_residency(ids, num_experts=8, hot_per_layer=8, policy="decayed", initial_order=order) + assert warm.misses == 0 and warm.miss_rate == 0.0 + # A cold start pays exactly one compulsory miss per distinct expert. + cold = simulate_residency(ids, num_experts=8, hot_per_layer=8, policy="decayed") + assert cold.misses == sum(len(np.unique(ids[:, layer])) for layer in range(2)) + + +def test_hand_checked_lru_and_static_counts(): + # One layer, top-1, two slots: A B A C A (experts 0 1 0 2 0) + ids = np.array([0, 1, 0, 2, 0]).reshape(5, 1, 1) + lru = simulate_residency(ids, num_experts=3, hot_per_layer=2, policy="lru") + assert lru.misses == 3 # cold 0, cold 1, then 2 evicts 1 -> 0 stays a hit + static = simulate_residency( + ids, num_experts=3, hot_per_layer=2, policy="static", initial_order=np.array([[3.0, 2.0, 1.0]]) + ) + assert static.misses == 1 # only expert 2 is outside the fixed set {0, 1} + + +def test_decayed_counts_keep_the_frequent_expert_through_one_off_scans(): + # Expert 0 is used often; pairs of one-off experts arrive between its uses. + # LRU evicts 0 on every scan; decayed counts keep it resident. + seq = [0] * 4 + [1, 2, 0, 3, 4, 0, 5, 6, 0] + ids = np.array(seq).reshape(-1, 1, 1) + lru = simulate_residency(ids, num_experts=7, hot_per_layer=2, policy="lru") + decayed = simulate_residency(ids, num_experts=7, hot_per_layer=2, policy="decayed", half_life=64) + assert decayed.misses == 7 # the first 0 and the six one-offs + assert lru.misses == 10 # plus three re-fetches of 0 + + +def test_reap_ordered_warm_start_beats_a_cold_start(): + ids, _ = _routes(300, 3, 4, 32, seed=1) + order = -np.tile(np.arange(32, dtype=float), (3, 1)) # expert 0 most salient + cold = simulate_residency(ids, num_experts=32, hot_per_layer=12, policy="decayed") + warm = simulate_residency( + ids, num_experts=32, hot_per_layer=12, policy="decayed", initial_order=order + ) + assert warm.misses < cold.misses + + +def test_fetch_threshold_drops_light_misses_and_reports_their_mass(): + ids, gates = _routes(200, 2, 4, 32, seed=2) + order = -np.tile(np.arange(32, dtype=float), (2, 1)) + exact = simulate_residency( + ids, num_experts=32, hot_per_layer=8, policy="static", initial_order=order, gate_weights=gates + ) + hybrid = simulate_residency( + ids, + num_experts=32, + hot_per_layer=8, + policy="static", + initial_order=order, + gate_weights=gates, + fetch_threshold=0.2, + ) + assert hybrid.misses + hybrid.dropped == exact.misses + assert hybrid.misses < exact.misses + assert 0.0 < hybrid.dropped_gate_mass_fraction < 1.0 + assert exact.dropped == 0 and exact.dropped_gate_mass_fraction == 0.0 + + +def test_warmup_tokens_are_replayed_but_not_counted(): + ids, _ = _routes(100, 1, 2, 16, seed=3) + result = simulate_residency(ids, num_experts=16, hot_per_layer=4, policy="lru", warmup_tokens=40) + assert result.tokens == 60 and result.routed_slots == 120 + + +def test_sessions_reset_and_pool_by_token_count(): + a, ga = _routes(80, 2, 3, 16, seed=4) + b, gb = _routes(40, 2, 3, 16, seed=5) + pooled = simulate_sessions([(a, ga), (b, gb)], num_experts=16, hot_per_layer=6, policy="lru", expert_bytes=1e6) + one = simulate_residency(a, num_experts=16, hot_per_layer=6, policy="lru") + two = simulate_residency(b, num_experts=16, hot_per_layer=6, policy="lru") + assert pooled.misses == one.misses + two.misses + assert pooled.tokens == 120 + assert pooled.ssd_bytes_per_token == pytest.approx(pooled.misses_per_token * 1e6) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"hot_per_layer": 0}, + {"hot_per_layer": 2}, # fewer than top-k + {"policy": "fifo"}, + {"policy": "static"}, # needs an order + {"fetch_threshold": 0.1}, # needs gates + ], +) +def test_invalid_replays_are_refused(kwargs): + ids, _ = _routes(10, 1, 3, 8) + params = {"num_experts": 8, "hot_per_layer": 4, "policy": "lru", **kwargs} + with pytest.raises(ValueError): + simulate_residency(ids, **params) + + +def test_report_applies_the_kill_rule_per_hot_size(): + ids, _ = _routes(200, 2, 4, 32, seed=6) + small = simulate_residency(ids, num_experts=32, hot_per_layer=6, policy="lru") + full = simulate_residency(ids, num_experts=32, hot_per_layer=32, policy="lru") + report = residency_report([small, full], kill_miss_rate=0.05) + assert report["by_hot_per_layer"]["32"]["verdict"] == "PASS" + assert report["by_hot_per_layer"]["6"]["verdict"] == "KILL" + + +def test_decode_rate_estimate(): + # 30 ms resident, 15 misses x 9 MB at 5 GB/s = 27 ms more + rate = decode_tokens_per_second( + resident_ms_per_token=30.0, misses_per_token=15, expert_bytes=9e6, ssd_bytes_per_second=5e9 + ) + assert rate == pytest.approx(1e3 / 57.0) + + +def test_route_recorder_keeps_the_first_tokens_in_order(): + recorder = RouteRecorder(layers=[3, 7], max_tokens=5) + for layer in (3, 7): + recorder.add_chunk(layer=layer, expert_ids=np.arange(6).reshape(3, 2), gate_weights=np.full((3, 2), 0.5)) + recorder.add_chunk(layer=layer, expert_ids=np.arange(6, 12).reshape(3, 2), gate_weights=np.full((3, 2), 0.25)) + arrays = recorder.as_arrays() + ids = arrays["routes__expert_ids"] + assert ids.shape == (5, 2, 2) and ids.dtype == np.int16 + np.testing.assert_array_equal(ids[:, 0].ravel(), np.arange(10)) + assert arrays["routes__gate_weights"][4, 1, 0] == np.float16(0.25) + + +def test_replay_cli_on_recorded_calibration_sessions(tmp_path, capsys): + from benchmarks import replay_dsv4_expert_residency as cli + from keep.quality.reap import ReapStats + + sessions = tmp_path / "cal" / "sessions" + sessions.mkdir(parents=True) + counts = np.zeros((2, 16), dtype=np.int64) + for index in range(2): + ids, gates = _routes(60, 2, 2, 16, seed=10 + index) + np.savez( + sessions / f"s{index}.npz", + routes__layers=np.array([0, 1]), + routes__expert_ids=ids.astype(np.int16), + routes__gate_weights=gates.astype(np.float16), + ) + for layer in range(2): + counts[layer] += np.bincount(ids[:, layer].ravel(), minlength=16) + stats = ReapStats( + layers=[0, 1], + num_experts=16, + route_count=counts, + gate_sum=counts * 0.5, + gate_norm_sum=counts * 1.0, + norm_sum=counts * 2.0, + norm_max=np.ones((2, 16)), + abs_max=np.ones((2, 16)), + routed_slots=counts.sum(axis=1), + ) + stats.save(tmp_path / "reap-stats.npz") + out = tmp_path / "residency.json" + assert ( + cli.main( + [ + "--calibration-dir", str(tmp_path / "cal"), + "--reap-stats", str(tmp_path / "reap-stats.npz"), + "--hot", "4,16", + "--fetch-threshold", "0.3", + "--resident-ms", "30", + "--output", str(out), + ] + ) + == 0 + ) + report = json.loads(out.read_text()) + assert report["sessions"] == 2 + assert report["by_hot_per_layer"]["16"]["verdict"] == "PASS" + assert {r["policy"] for r in report["results"]} == {"static", "lru", "decayed"} + assert any(key.endswith("+drop<0.3") for key in report["tokens_per_second_estimate"]) From 5baed30da02a24a843c5f5597f17aab21d212681 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:18:16 +0000 Subject: [PATCH 05/74] ramp: opt-in one-dispatch sinkhorn for DeepSeek-V4 hyper-connections Every hyper-connection turns hc_mult x hc_mult mixing logits into a doubly-stochastic matrix through a row softmax and 19 more rounds of row/column normalisation. The model's compiled path still launches a few kernels per round, because each normalisation is a reduction, and it runs 86 times per decoded V4-Flash token. The roadmap measured 30-70 ms of sinkhorn per token. ramp.kernels.hc_sinkhorn.fused_hc_sinkhorn does the whole sinkhorn in one Metal dispatch, one thread per matrix, with the values in registers, following the compiled path's operation order. exp and summation order are not guaranteed bit-identical, so it is opt-in: RAMP_HC_SINKHORN=metal selects it in HyperConnection (also used by the GLM-5 next adapter), and the default path is untouched. _hc_split_logits_ops computes pre, post and the comb logits exactly as the compiled path does. hc_sinkhorn_reference states the algorithm in float32 NumPy; a test pins it to the compiled path on CPU. Metal-only tests compare the fused kernel with the compiled path and HyperConnection's outputs with the flag on and off; they skip without a Metal GPU. benchmarks/bench_hc_sinkhorn.py is the kill test: KILL if the fused path is under 3x faster at decode shape or differs by more than 2e-6 relative. --- benchmarks/bench_hc_sinkhorn.py | 123 ++++++++++++++ src/ramp/kernels/hc_sinkhorn.py | 163 +++++++++++++++++++ src/ramp/models/deepseek_v4_flash_adapter.py | 55 ++++++- tests/test_hc_sinkhorn.py | 101 ++++++++++++ 4 files changed, 434 insertions(+), 8 deletions(-) create mode 100644 benchmarks/bench_hc_sinkhorn.py create mode 100644 src/ramp/kernels/hc_sinkhorn.py create mode 100644 tests/test_hc_sinkhorn.py diff --git a/benchmarks/bench_hc_sinkhorn.py b/benchmarks/bench_hc_sinkhorn.py new file mode 100644 index 0000000..f56fdb7 --- /dev/null +++ b/benchmarks/bench_hc_sinkhorn.py @@ -0,0 +1,123 @@ +"""Kill test: the one-dispatch hyper-connection sinkhorn against the compiled path. + +Times the model's compiled ``_hc_split_sinkhorn_ops`` against the logits half +plus :func:`ramp.kernels.hc_sinkhorn.fused_hc_sinkhorn`, at decode shape (one +row) and a prefill chunk, and reports milliseconds per DeepSeek-V4-Flash +decode step (86 calls: two hyper-connections in each of 43 blocks). + +KILL if the fused path is less than 3x faster at decode shape, or if it +differs from the compiled path by more than 2e-6 relative. If it passes, set +``RAMP_HC_SINKHORN=metal`` for the decode headline. + + uv run python benchmarks/bench_hc_sinkhorn.py --output-json artifacts/benchmarks/hc-sinkhorn-m4max.json +""" + +from __future__ import annotations + +import argparse +import json +import platform +import sys +import time +from collections.abc import Sequence +from pathlib import Path + +import mlx.core as mx +import numpy as np + +from ramp.kernels.hc_sinkhorn import fused_hc_sinkhorn +from ramp.models.deepseek_v4_flash_adapter import _hc_split_logits_ops, _hc_split_sinkhorn_ops + +CALLS_PER_DECODE_STEP = 86 +MIN_DECODE_SPEEDUP = 3.0 +MAX_RELATIVE_DIFF = 2e-6 + + +def _per_call_ms(fn, iterations: int, warmup: int) -> float: + for _ in range(warmup): + mx.eval(fn()) + started = time.perf_counter() + for _ in range(iterations): + mx.eval(fn()) + return (time.perf_counter() - started) * 1e3 / iterations + + +def measure(rows: int, *, hc: int, iters: int, eps: float, iterations: int, warmup: int, seed: int) -> dict: + rng = np.random.default_rng(seed) + mix = (2 + hc) * hc + mixes = mx.array(rng.normal(size=(1, rows, mix)).astype(np.float32) * 3) + scale = mx.array(rng.uniform(0.5, 2.0, size=(3,)).astype(np.float32)) + base = mx.array(rng.normal(size=(mix,)).astype(np.float32)) + + def compiled(): + return _hc_split_sinkhorn_ops(mixes, scale, base, hc, iters, eps) + + def fused(): + pre, post, logits = _hc_split_logits_ops(mixes, scale, base, hc, eps) + return pre, post, fused_hc_sinkhorn(logits, iters=iters, eps=eps) + + want, got = np.array(compiled()[2]), np.array(fused()[2]) + rel = float(np.max(np.abs(got - want) / np.maximum(np.abs(want), 1e-12))) + t_compiled = _per_call_ms(compiled, iterations, warmup) + t_fused = _per_call_ms(fused, iterations, warmup) + return { + "rows": rows, + "compiled_ms_per_call": t_compiled, + "fused_ms_per_call": t_fused, + "speedup": t_compiled / t_fused, + "compiled_ms_per_decode_step": t_compiled * CALLS_PER_DECODE_STEP, + "fused_ms_per_decode_step": t_fused * CALLS_PER_DECODE_STEP, + "max_relative_diff": rel, + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--rows", default="1,1024", help="matrices per call (1 = decode)") + parser.add_argument("--hc-mult", type=int, default=4) + parser.add_argument("--iters", type=int, default=20) + parser.add_argument("--eps", type=float, default=1e-6) + parser.add_argument("--iterations", type=int, default=500) + parser.add_argument("--warmup", type=int, default=50) + parser.add_argument("--seed", type=int, default=20261105) + parser.add_argument("--output-json", type=Path) + args = parser.parse_args(argv) + + on_gpu = mx.default_device() == mx.gpu + rows = [ + measure( + int(r), + hc=args.hc_mult, + iters=args.iters, + eps=args.eps, + iterations=args.iterations, + warmup=args.warmup, + seed=args.seed, + ) + for r in args.rows.split(",") + ] + decode = next((row for row in rows if row["rows"] == 1), rows[0]) + reasons = [] + if any(row["max_relative_diff"] > MAX_RELATIVE_DIFF for row in rows): + reasons.append(f"fused output differs by more than {MAX_RELATIVE_DIFF} relative") + if decode["speedup"] < MIN_DECODE_SPEEDUP: + reasons.append(f"decode-shape speedup {decode['speedup']:.2f}x is under {MIN_DECODE_SPEEDUP}x") + result = { + "record_type": "ramp_hc_sinkhorn_kill_test_v1", + "device": str(mx.default_device()), + "mlx_version": mx.__version__, + "platform": platform.platform(), + "rows": rows, + "verdict": "KILL" if reasons else ("PASS" if on_gpu else "NOT_ON_GPU"), + "reasons": reasons, + } + text = json.dumps(result, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(text + "\n") + print(text) + return 1 if result["verdict"] == "KILL" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/ramp/kernels/hc_sinkhorn.py b/src/ramp/kernels/hc_sinkhorn.py new file mode 100644 index 0000000..267e6ce --- /dev/null +++ b/src/ramp/kernels/hc_sinkhorn.py @@ -0,0 +1,163 @@ +"""One-dispatch Sinkhorn normalisation for DeepSeek-V4 hyper-connections. + +Every hyper-connection (two per block: 86 calls per decoded token on +DeepSeek-V4-Flash) turns ``hc_mult x hc_mult`` mixing logits into a doubly +stochastic matrix: a row softmax, a column normalisation, then +``sinkhorn_iters - 1`` alternating row and column normalisations. The +compiled MLX path still launches a few kernels per iteration because each +normalisation is a reduction, so a 4x4 matrix costs dozens of dispatches per +call. The roadmap measured 30 to 70 ms of sinkhorn per decoded token. + +:func:`fused_hc_sinkhorn` does the whole thing in one Metal dispatch, one +thread per matrix, holding the 16 values in registers. It follows the MLX +path's operation order, but exp and summation order are not guaranteed +bit-identical, so it is **opt-in** (``RAMP_HC_SINKHORN=metal``) until the +parity and speed check in ``benchmarks/bench_hc_sinkhorn.py`` passes on the +target Mac. + +:func:`hc_sinkhorn_reference` is a float32 NumPy statement of the same +algorithm; tests pin it against the model's compiled path, and the Metal +kernel against both. +""" + +from __future__ import annotations + +import os +from functools import cache + +import numpy as np + +__all__ = [ + "HC_SINKHORN_ENV", + "fused_hc_sinkhorn", + "fused_hc_sinkhorn_enabled", + "hc_sinkhorn_reference", +] + +HC_SINKHORN_ENV = "RAMP_HC_SINKHORN" +_BACKENDS = ("mlx", "metal") + +_SOURCE = """ + uint row = thread_position_in_grid.x; + if (row >= rows[0]) { + return; + } + const uint n = HC * HC; + const uint base = row * n; + const float e = eps[0]; + float m[HC * HC]; + for (uint k = 0; k < n; ++k) { + m[k] = static_cast(logits[base + k]); + } + // Row softmax (max-subtracted), then + eps. + for (uint i = 0; i < HC; ++i) { + float peak = m[i * HC]; + for (uint j = 1; j < HC; ++j) { + peak = metal::max(peak, m[i * HC + j]); + } + float total = 0.0f; + for (uint j = 0; j < HC; ++j) { + float v = metal::precise::exp(m[i * HC + j] - peak); + m[i * HC + j] = v; + total += v; + } + for (uint j = 0; j < HC; ++j) { + m[i * HC + j] = m[i * HC + j] / total + e; + } + } + // Column normalisation. + for (uint j = 0; j < HC; ++j) { + float total = 0.0f; + for (uint i = 0; i < HC; ++i) { + total += m[i * HC + j]; + } + for (uint i = 0; i < HC; ++i) { + m[i * HC + j] = m[i * HC + j] / (total + e); + } + } + // ITERS - 1 alternating row / column normalisations. + for (uint it = 1; it < ITERS; ++it) { + for (uint i = 0; i < HC; ++i) { + float total = 0.0f; + for (uint j = 0; j < HC; ++j) { + total += m[i * HC + j]; + } + for (uint j = 0; j < HC; ++j) { + m[i * HC + j] = m[i * HC + j] / (total + e); + } + } + for (uint j = 0; j < HC; ++j) { + float total = 0.0f; + for (uint i = 0; i < HC; ++i) { + total += m[i * HC + j]; + } + for (uint i = 0; i < HC; ++i) { + m[i * HC + j] = m[i * HC + j] / (total + e); + } + } + } + for (uint k = 0; k < n; ++k) { + comb[base + k] = m[k]; + } +""" + + +def fused_hc_sinkhorn_enabled() -> bool: + """Whether ``RAMP_HC_SINKHORN=metal`` selects the fused kernel.""" + + value = os.environ.get(HC_SINKHORN_ENV, "mlx").strip().lower() or "mlx" + if value not in _BACKENDS: + raise ValueError(f"{HC_SINKHORN_ENV} must be one of {_BACKENDS}, got {value!r}") + return value == "metal" + + +@cache +def _kernel(): + import mlx.core as mx + + return mx.fast.metal_kernel( + name="ramp_hc_sinkhorn", + input_names=["logits", "eps", "rows"], + output_names=["comb"], + source=_SOURCE, + ) + + +def fused_hc_sinkhorn(logits, *, iters: int, eps: float): + """Doubly-stochastic ``[..., hc, hc]`` mixing matrices in one dispatch. + + ``logits`` are the scaled, biased comb logits (float32); ``iters`` is the + model's ``hc_sinkhorn_iters``. Returns float32 of the same shape. + """ + + import mlx.core as mx + + if logits.ndim < 2 or logits.shape[-1] != logits.shape[-2]: + raise ValueError(f"expected [..., hc, hc] logits, got {tuple(logits.shape)}") + if iters < 1: + raise ValueError("iters must be at least 1") + hc = int(logits.shape[-1]) + rows = int(np.prod(logits.shape[:-2])) if logits.ndim > 2 else 1 + flat = mx.contiguous(logits.astype(mx.float32).reshape(rows, hc * hc)) + (out,) = _kernel()( + inputs=[flat, mx.array([eps], dtype=mx.float32), mx.array([rows], dtype=mx.uint32)], + template=[("HC", hc), ("ITERS", int(iters))], + grid=(rows, 1, 1), + threadgroup=(min(rows, 64), 1, 1), + output_shapes=[(rows, hc * hc)], + output_dtypes=[mx.float32], + ) + return out.reshape(logits.shape) + + +def hc_sinkhorn_reference(logits: np.ndarray, *, iters: int, eps: float) -> np.ndarray: + """The model's sinkhorn in float32 NumPy, step for step.""" + + m = np.asarray(logits, dtype=np.float32) + m = np.exp(m - m.max(axis=-1, keepdims=True), dtype=np.float32) + m = m / m.sum(axis=-1, keepdims=True, dtype=np.float32) + np.float32(eps) + m = m / (m.sum(axis=-2, keepdims=True, dtype=np.float32) + np.float32(eps)) + for _ in range(max(int(iters) - 1, 0)): + m = m / (m.sum(axis=-1, keepdims=True, dtype=np.float32) + np.float32(eps)) + m = m / (m.sum(axis=-2, keepdims=True, dtype=np.float32) + np.float32(eps)) + return m.astype(np.float32) diff --git a/src/ramp/models/deepseek_v4_flash_adapter.py b/src/ramp/models/deepseek_v4_flash_adapter.py index d2bffcb..add7193 100644 --- a/src/ramp/models/deepseek_v4_flash_adapter.py +++ b/src/ramp/models/deepseek_v4_flash_adapter.py @@ -149,6 +149,7 @@ repack_fp8_block_as_mxfp8, ) from keep.io.load import load_quantized_vq_switch_linear +from ramp.kernels.hc_sinkhorn import fused_hc_sinkhorn, fused_hc_sinkhorn_enabled from keep.io.source_safetensors import read_safetensors_file_header from ramp.models.glm4_moe_adapter import QuantizedVQSwitchGLU @@ -807,6 +808,34 @@ def _hc_split_sinkhorn_ops( return pre, post, comb +@mx.compile +def _hc_split_logits_ops( + mixes: mx.array, + scale: mx.array, + base: mx.array, + hc_mult: int, + eps: float, +) -> tuple[mx.array, mx.array, mx.array]: + """``_hc_split_sinkhorn_ops`` up to the comb logits, for the fused sinkhorn. + + ``pre`` and ``post`` are computed exactly as there; only the sinkhorn on + the returned logits moves to :func:`ramp.kernels.hc_sinkhorn.fused_hc_sinkhorn`. + """ + + mixes = mixes.astype(mx.float32) + scale = scale.astype(mx.float32) + base = base.astype(mx.float32) + pre_scale, post_scale, comb_scale = scale[0], scale[1], scale[2] + pre = mx.sigmoid(mixes[..., :hc_mult] * pre_scale + base[:hc_mult]) + eps + post = 2 * mx.sigmoid( + mixes[..., hc_mult : 2 * hc_mult] * post_scale + base[hc_mult : 2 * hc_mult] + ) + logits = mixes[..., 2 * hc_mult :].reshape( + *mixes.shape[:-1], hc_mult, hc_mult + ) * comb_scale + base[2 * hc_mult :].reshape(hc_mult, hc_mult) + return pre, post, logits + + class HyperConnection(nn.Module): def __init__(self, config: DeepseekV4FlashVQModelArgs): super().__init__() @@ -824,14 +853,24 @@ def __call__(self, x: mx.array): y = x.astype(mx.float32) z = mx.fast.rms_norm(y.flatten(-2), None, self.norm_eps) mixes = z @ self.fn.T - pre, post, comb = _hc_split_sinkhorn_ops( - mixes, - self.scale, - self.base, - self.hc_mult, - self.sinkhorn_iters, - self.hc_eps, - ) + if fused_hc_sinkhorn_enabled(): + # Opt-in (RAMP_HC_SINKHORN=metal): one dispatch for the whole + # sinkhorn instead of a few per iteration. Not bit-identical. + pre, post, logits = _hc_split_logits_ops( + mixes, self.scale, self.base, self.hc_mult, self.hc_eps + ) + comb = fused_hc_sinkhorn( + logits, iters=self.sinkhorn_iters, eps=self.hc_eps + ) + else: + pre, post, comb = _hc_split_sinkhorn_ops( + mixes, + self.scale, + self.base, + self.hc_mult, + self.sinkhorn_iters, + self.hc_eps, + ) return (pre[..., None] * y).sum(axis=2).astype(x.dtype), post, comb diff --git a/tests/test_hc_sinkhorn.py b/tests/test_hc_sinkhorn.py new file mode 100644 index 0000000..d5ac829 --- /dev/null +++ b/tests/test_hc_sinkhorn.py @@ -0,0 +1,101 @@ +"""The opt-in one-dispatch hyper-connection sinkhorn and its reference.""" + +from __future__ import annotations + +import numpy as np +import pytest + +mx = pytest.importorskip("mlx.core") + +from ramp.kernels.hc_sinkhorn import ( # noqa: E402 + HC_SINKHORN_ENV, + fused_hc_sinkhorn, + fused_hc_sinkhorn_enabled, + hc_sinkhorn_reference, +) +from ramp.models.deepseek_v4_flash_adapter import ( # noqa: E402 + _hc_split_logits_ops, + _hc_split_sinkhorn_ops, +) + +HC, ITERS, EPS = 4, 20, 1e-6 + + +def _inputs(rows, seed=0): + rng = np.random.default_rng(seed) + mix = (2 + HC) * HC + mixes = mx.array(rng.normal(size=(1, rows, mix)).astype(np.float32) * 3) + scale = mx.array(rng.uniform(0.5, 2.0, size=(3,)).astype(np.float32)) + base = mx.array(rng.normal(size=(mix,)).astype(np.float32)) + return mixes, scale, base + + +def _metal_available() -> bool: + metal = getattr(mx, "metal", None) + return bool(metal is not None and metal.is_available()) + + +@pytest.mark.parametrize("rows", [1, 7, 256]) +def test_reference_reproduces_the_models_compiled_sinkhorn(rows): + mixes, scale, base = _inputs(rows) + pre, post, comb = _hc_split_sinkhorn_ops(mixes, scale, base, HC, ITERS, EPS) + pre2, post2, logits = _hc_split_logits_ops(mixes, scale, base, HC, EPS) + # pre/post are computed identically on both paths. + assert mx.array_equal(pre, pre2) and mx.array_equal(post, post2) + want = np.array(comb) + got = hc_sinkhorn_reference(np.array(logits), iters=ITERS, eps=EPS) + np.testing.assert_allclose(got, want, rtol=2e-6, atol=1e-7) + # The last step is a column normalisation, so columns sum to one; rows + # only approach it (20 rounds can leave them ~0.3% off for some inputs). + np.testing.assert_allclose(got.sum(axis=-2), 1.0, atol=1e-5) + + +def test_backend_flag(monkeypatch): + monkeypatch.delenv(HC_SINKHORN_ENV, raising=False) + assert fused_hc_sinkhorn_enabled() is False + monkeypatch.setenv(HC_SINKHORN_ENV, "metal") + assert fused_hc_sinkhorn_enabled() is True + monkeypatch.setenv(HC_SINKHORN_ENV, "cuda") + with pytest.raises(ValueError, match=HC_SINKHORN_ENV): + fused_hc_sinkhorn_enabled() + + +def test_fused_kernel_validates_its_input_shape(): + with pytest.raises(ValueError, match="hc, hc"): + fused_hc_sinkhorn(mx.zeros((3, 4, 5)), iters=ITERS, eps=EPS) + with pytest.raises(ValueError, match="iters"): + fused_hc_sinkhorn(mx.zeros((3, 4, 4)), iters=0, eps=EPS) + + +@pytest.mark.skipif(not _metal_available(), reason="needs a Metal GPU") +@pytest.mark.parametrize("rows", [1, 7, 1024]) +def test_fused_kernel_matches_the_compiled_path(rows): + mixes, scale, base = _inputs(rows, seed=rows) + _, _, comb = _hc_split_sinkhorn_ops(mixes, scale, base, HC, ITERS, EPS) + _, _, logits = _hc_split_logits_ops(mixes, scale, base, HC, EPS) + fused = fused_hc_sinkhorn(logits, iters=ITERS, eps=EPS) + assert fused.shape == comb.shape and fused.dtype == mx.float32 + np.testing.assert_allclose(np.array(fused), np.array(comb), rtol=2e-6, atol=1e-7) + + +@pytest.mark.skipif(not _metal_available(), reason="needs a Metal GPU") +def test_hyper_connection_output_with_the_flag(monkeypatch): + from ramp.models.deepseek_v4_flash_adapter import ( + DeepseekV4FlashVQModelArgs, + HyperConnection, + ) + + args = DeepseekV4FlashVQModelArgs(hidden_size=64, hc_mult=HC, num_hidden_layers=1, compress_ratios=[0]) + hc = HyperConnection(args) + rng = np.random.default_rng(1) + hc.fn = mx.array(rng.normal(size=hc.fn.shape).astype(np.float32) * 0.05) + hc.base = mx.array(rng.normal(size=hc.base.shape).astype(np.float32)) + x = mx.array(rng.normal(size=(1, 3, HC, 64)).astype(np.float32)).astype(mx.bfloat16) + monkeypatch.delenv(HC_SINKHORN_ENV, raising=False) + want = hc(x) + monkeypatch.setenv(HC_SINKHORN_ENV, "metal") + got = hc(x) + for a, b in zip(got, want): + np.testing.assert_allclose( + np.array(a.astype(mx.float32)), np.array(b.astype(mx.float32)), rtol=1e-2, atol=1e-6 + ) From 00580c4bf3adbce2a5e8164a89e0bc1ced0c46a9 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:23:13 +0000 Subject: [PATCH 06/74] ramp: opt-in single-token E8P kernels for 16-bit codes on GPUs without NAX The shipped DeepSeek-V4-Flash artifact uses 16-bit E8P codes. Every M=1 fast path in QuantizedVQSwitchLinear is gated on 8-bit codes, and the E8P tensor-unit kernels are M5-only (RAMP refuses them on an M4 Max, where 36 of 94 kernel tests are wrong). So on M1-M4 every decoded token's routed projections fall through to the scalar kernel, which spends a whole threadgroup per output element. gather_vqmm_m1.metal already had an E8P branch that nothing dispatched to. With RAMP_E8P_M1=1 (opt-in), M=1 calls with 16-bit codes, unsorted routes and the auto/direct strategy use: - e8p_m1_token_layout for gate/up: the existing M=1 kernel with CODE_BITS=16, one activation row against top-k experts; - e8p_m1_per_route for down: the same kernel source with the activation read offset by route. It is built as a separate kernel from a two-line text substitution, so the production 8-bit kernel is byte-identical. A test proves undoing the substitution restores the file. The flag is off by default. Tests pin the decision table and the dispatch wiring, and Metal-only tests compare both kernels with a dense decode of the same codes. benchmarks/bench_e8p_m1.py is the M4 kill test. It times one routed FFN at decode shape against today's dispatch, sweeps row packing and threadgroup codebook staging, and checks the outputs agree. Verdict: PASS at >= 200 GB/s effective, TUNE at 100-200 (next step: decode each codeword once instead of once per dimension), KILL under 100 (serve on the M5). Its synthetic projections reproduce the roadmap's 2,129,920 bytes per projection per expert. --- benchmarks/bench_e8p_m1.py | 211 +++++++++++++++++++++++++++++++++++ src/ramp/kernels/e8p_m1.py | 175 +++++++++++++++++++++++++++++ src/ramp/nn/switch_linear.py | 35 +++++- tests/test_e8p_m1.py | 145 ++++++++++++++++++++++++ 4 files changed, 564 insertions(+), 2 deletions(-) create mode 100644 benchmarks/bench_e8p_m1.py create mode 100644 src/ramp/kernels/e8p_m1.py create mode 100644 tests/test_e8p_m1.py diff --git a/benchmarks/bench_e8p_m1.py b/benchmarks/bench_e8p_m1.py new file mode 100644 index 0000000..5abe0d7 --- /dev/null +++ b/benchmarks/bench_e8p_m1.py @@ -0,0 +1,211 @@ +"""Kill test: single-token E8P (16-bit codes) on a GPU without the NAX kernels. + +Times one routed FFN at decode shape (gate, up and down for top-6 experts) two +ways in the same process: + +- ``production``: today's dispatch (the scalar kernel on M1-M4; NAX on M5); +- ``e8p_m1``: ``RAMP_E8P_M1=1``, the M=1 E8P kernels, swept over row packing + and threadgroup codebook staging. + +Effective bandwidth is the expert bytes one FFN reads (6 experts x 3 +projections of 2-bit codes plus their scales) over the median time. Verdict +for the best ``e8p_m1`` configuration: **PASS** at >= 200 GB/s, **TUNE** at +100-200 (decode the codeword once instead of per dimension, then re-measure), +**KILL** under 100 (serve on the M5 instead). Outputs of the two paths are +compared on every configuration. + +Synthetic codes at DeepSeek-V4-Flash shapes need no download; ``--artifact-dir`` +uses one real layer of the shipped artifact instead:: + + RAMP_E8P_M1=0 uv run python benchmarks/bench_e8p_m1.py --output-json artifacts/benchmarks/e8p-m1-m4max.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import sys +import time +from collections.abc import Sequence +from pathlib import Path + +import mlx.core as mx +import numpy as np + +from keep.vq.e8 import e8p_packed_abs_grid +from ramp.kernels import e8p_m1 +from ramp.nn.switch_linear import QuantizedVQSwitchLinear + +PROJECTIONS = ("gate_proj", "up_proj", "down_proj") +PASS_GB_S = 200.0 +KILL_GB_S = 100.0 + + +def synthetic_projection(*, experts: int, out: int, inp: int, group_size: int, seed: int) -> QuantizedVQSwitchLinear: + rng = np.random.default_rng(seed) + codes = rng.integers(0, 1 << 16, size=(experts, out, inp // 8), dtype=np.uint16) + scales = rng.uniform(0.01, 0.05, size=(experts, out, inp // group_size)).astype(np.float16) + return QuantizedVQSwitchLinear( + input_dims=inp, + output_dims=out, + num_experts=experts, + codes=mx.array(codes), + scales=mx.array(scales), + codebook=mx.array(e8p_packed_abs_grid()), + group_size=group_size, + code_bits=16, + ) + + +def projection_bytes_per_expert(layer: QuantizedVQSwitchLinear) -> int: + per_expert = layer.codes.nbytes // layer.num_experts + layer.scales.nbytes // layer.num_experts + return int(per_expert) + + +def _ffn(layers, x, routes, intermediate): + gate = layers["gate_proj"](x, routes) + up = layers["up_proj"](x, routes) + act = (gate * mx.sigmoid(gate) * up).astype(x.dtype) # [1, k, I] + return layers["down_proj"](act.reshape(1, routes.shape[-1], intermediate), routes) + + +def _median_ms(fn, routes_list, iterations, warmup) -> float: + for i in range(warmup): + mx.eval(fn(routes_list[i % len(routes_list)])) + samples = [] + for i in range(iterations): + started = time.perf_counter() + mx.eval(fn(routes_list[i % len(routes_list)])) + samples.append((time.perf_counter() - started) * 1e3) + return statistics.median(samples) + + +def run(layers, *, top_k: int, iterations: int, warmup: int, seed: int, sweep: Sequence[tuple[int, bool]]) -> dict: + rng = np.random.default_rng(seed) + experts = layers["gate_proj"].num_experts + hidden = layers["gate_proj"].input_dims + intermediate = layers["gate_proj"].output_dims + x = mx.array(rng.normal(size=(1, hidden)).astype(np.float32)).astype(mx.float16) + # Fresh routes per call so consecutive calls do not reuse cached experts. + routes_list = [ + mx.array(np.sort(rng.choice(experts, size=top_k, replace=False))[None].astype(np.uint32)) + for _ in range(64) + ] + ffn_bytes = top_k * sum(projection_bytes_per_expert(layers[p]) for p in PROJECTIONS) + + os.environ[e8p_m1.E8P_M1_ENV] = "0" + production_ms = _median_ms(lambda r: _ffn(layers, x, r, intermediate), routes_list, iterations, warmup) + reference = [np.array(_ffn(layers, x, r, intermediate).astype(mx.float32)) for r in routes_list[:4]] + + configs = [] + os.environ[e8p_m1.E8P_M1_ENV] = "1" + original_token, original_route = e8p_m1.e8p_m1_token_layout, e8p_m1.e8p_m1_per_route + import ramp.nn.switch_linear as switch_linear + + try: + for rows_per_tg, tg_codebook in sweep: + switch_linear.e8p_m1_token_layout = ( + lambda *a, _r=rows_per_tg, _c=tg_codebook, **k: original_token( + *a, rows_per_threadgroup=_r, use_threadgroup_codebook=_c, **k + ) + ) + switch_linear.e8p_m1_per_route = ( + lambda *a, _r=rows_per_tg, _c=tg_codebook, **k: original_route( + *a, rows_per_threadgroup=_r, use_threadgroup_codebook=_c, **k + ) + ) + got = [np.array(_ffn(layers, x, r, intermediate).astype(mx.float32)) for r in routes_list[:4]] + diff = max( + float(np.max(np.abs(g - w)) / max(float(np.max(np.abs(w))), 1e-12)) + for g, w in zip(got, reference) + ) + ms = _median_ms(lambda r: _ffn(layers, x, r, intermediate), routes_list, iterations, warmup) + configs.append( + { + "rows_per_threadgroup": rows_per_tg, + "threadgroup_codebook": tg_codebook, + "median_ms_per_ffn": ms, + "effective_gb_per_s": ffn_bytes / (ms / 1e3) / 1e9, + "speedup_over_production": production_ms / ms, + "max_relative_diff_vs_production": diff, + } + ) + finally: + switch_linear.e8p_m1_token_layout = original_token + switch_linear.e8p_m1_per_route = original_route + os.environ.pop(e8p_m1.E8P_M1_ENV, None) + + best = max(configs, key=lambda c: c["effective_gb_per_s"]) + agrees = all(c["max_relative_diff_vs_production"] <= 1e-2 for c in configs) + gb_s = best["effective_gb_per_s"] + verdict = "KILL" if not agrees or gb_s < KILL_GB_S else ("PASS" if gb_s >= PASS_GB_S else "TUNE") + return { + "ffn_bytes": ffn_bytes, + "production_median_ms_per_ffn": production_ms, + "production_effective_gb_per_s": ffn_bytes / (production_ms / 1e3) / 1e9, + "configs": configs, + "best": best, + "outputs_agree": agrees, + "verdict": verdict if mx.default_device() == mx.gpu else "NOT_ON_GPU", + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--artifact-dir", type=Path, help="real E8P artifact; default synthetic") + parser.add_argument("--layer", type=int, default=20) + parser.add_argument("--experts", type=int, default=64, help="synthetic experts per projection") + parser.add_argument("--hidden", type=int, default=4096) + parser.add_argument("--intermediate", type=int, default=2048) + parser.add_argument("--group-size", type=int, default=512) + parser.add_argument("--top-k", type=int, default=6) + parser.add_argument("--iterations", type=int, default=200) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--seed", type=int, default=20261105) + parser.add_argument("--sweep", default="8:1,16:1,32:1,64:1,32:0", help="rows_per_threadgroup:tg_codebook,...") + parser.add_argument("--output-json", type=Path) + args = parser.parse_args(argv) + + if args.artifact_dir is not None: + from keep.io.load import load_quantized_vq_switch_linear + + layers = { + p: load_quantized_vq_switch_linear( + args.artifact_dir.expanduser() / f"layer-{args.layer:05d}-{p}.safetensors", + f"model.layers.{args.layer}.ffn.switch_mlp.{p}", + ) + for p in PROJECTIONS + } + if any(layer.code_bits != 16 for layer in layers.values()): + parser.error("the artifact layer is not 16-bit E8P") + else: + layers = { + "gate_proj": synthetic_projection(experts=args.experts, out=args.intermediate, inp=args.hidden, group_size=args.group_size, seed=1), + "up_proj": synthetic_projection(experts=args.experts, out=args.intermediate, inp=args.hidden, group_size=args.group_size, seed=2), + "down_proj": synthetic_projection(experts=args.experts, out=args.hidden, inp=args.intermediate, group_size=args.group_size, seed=3), + } + sweep = [] + for item in args.sweep.split(","): + rows, codebook = item.split(":") + sweep.append((int(rows), codebook.strip() == "1")) + result = { + "record_type": "ramp_e8p_m1_kill_test_v1", + "device": str(mx.default_device()), + "mlx_version": mx.__version__, + "platform": platform.platform(), + "source": "synthetic" if args.artifact_dir is None else str(args.artifact_dir), + **run(layers, top_k=args.top_k, iterations=args.iterations, warmup=args.warmup, seed=args.seed, sweep=sweep), + } + text = json.dumps(result, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(text + "\n") + print(text) + return 1 if result["verdict"] == "KILL" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/ramp/kernels/e8p_m1.py b/src/ramp/kernels/e8p_m1.py new file mode 100644 index 0000000..1f943a7 --- /dev/null +++ b/src/ramp/kernels/e8p_m1.py @@ -0,0 +1,175 @@ +"""Opt-in single-token (M=1) kernels for 16-bit E8P codes on GPUs without NAX. + +The shipped DeepSeek-V4-Flash artifact uses 16-bit E8P codes. Every M=1 fast +path in ``QuantizedVQSwitchLinear`` is gated on 8-bit codes, and the E8P +tensor-unit kernels are M5-only, so on an M1-M4 GPU a decoded token's routed +projections fall through to the scalar kernel, which spends a whole 256-thread +threadgroup per output element. ``gather_vqmm_m1.metal`` already carries an +E8P branch; nothing dispatched to it. + +With ``RAMP_E8P_M1=1`` (opt-in until ``benchmarks/bench_e8p_m1.py`` passes on +the target Mac), M=1 calls with 16-bit codes use: + +- :func:`e8p_m1_token_layout` for gate/up (one activation row, top-k + experts): the existing M=1 kernel with ``CODE_BITS=16``. +- :func:`e8p_m1_per_route` for down (one activation row *per route*): the + same kernel source with the activation read offset by route. It is built as + a separate kernel from a text substitution, so the production 8-bit kernel + stays byte-identical. + +The E8P branch decodes each codeword once per dimension. If the kill test +lands under its bandwidth bar, decoding a codeword once into eight values is +the first optimisation to try. +""" + +from __future__ import annotations + +import os +from functools import cache +from pathlib import Path + +__all__ = [ + "E8P_M1_ENV", + "e8p_m1_enabled", + "e8p_m1_per_route", + "e8p_m1_token_layout", + "per_route_x_source", + "should_use_e8p_m1", +] + +E8P_M1_ENV = "RAMP_E8P_M1" +_KERNEL_DIR = Path(__file__).resolve().parent +_ON = {"1", "on", "true", "yes"} +_OFF = {"", "0", "off", "false", "no"} +_X_INDEX_REWRITES = ( + ("uint x_base = cw * 8u;", "uint x_base = route * (codewords * 8u) + cw * 8u;"), + ("uint x_idx = cw * 8u + dim;", "uint x_idx = route * (codewords * 8u) + cw * 8u + dim;"), +) + + +def e8p_m1_enabled() -> bool: + value = os.environ.get(E8P_M1_ENV, "").strip().lower() + if value in _ON: + return True + if value in _OFF: + return False + raise ValueError(f"{E8P_M1_ENV} must be on/off (1/0), got {value!r}") + + +def should_use_e8p_m1( + *, code_bits: int, tokens: int, sorted_indices: bool, route_strategy: str +) -> bool: + """The opt-in decision, kept pure so the dispatch table is testable.""" + + return ( + code_bits == 16 + and tokens == 1 + and not sorted_indices + and route_strategy in ("auto", "direct") + and e8p_m1_enabled() + ) + + +def per_route_x_source() -> str: + """``gather_vqmm_m1.metal`` with the activation row chosen by route.""" + + source = (_KERNEL_DIR / "gather_vqmm_m1.metal").read_text() + for old, new in _X_INDEX_REWRITES: + count = source.count(old) + if count != 1: + raise RuntimeError( + f"gather_vqmm_m1.metal changed: expected one {old!r}, found {count}" + ) + source = source.replace(old, new) + return source + + +@cache +def _per_route_kernel(): + import mlx.core as mx + + return mx.fast.metal_kernel( + name="ramp_e8p_m1_per_route_x", + input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], + source=per_route_x_source(), + header=(_KERNEL_DIR / "common_e8.metal").read_text(), + ) + + +def _launch_shape(output_dims: int, rows_per_threadgroup: int) -> tuple[int, int]: + lanes_per_row = 256 // rows_per_threadgroup + if rows_per_threadgroup * lanes_per_row != 256 or lanes_per_row < 1: + raise ValueError("rows_per_threadgroup must divide 256") + return lanes_per_row, (output_dims + rows_per_threadgroup - 1) // rows_per_threadgroup + + +def e8p_m1_token_layout( + x, + codes, + scales, + codebook, + rhs_indices, + *, + output_dims: int, + rows_per_threadgroup: int = 32, + use_threadgroup_codebook: bool = True, +): + """One activation row against ``top_k`` experts: ``[1, top_k, out]``.""" + + from ramp.kernels.gather_vqmm import gather_vqmm_m1_kernel_unchecked + + if x.shape[0] != 1 or rhs_indices.ndim != 2 or rhs_indices.shape[0] != 1: + raise ValueError("token-layout E8P M=1 needs x [1, in] and rhs_indices [1, top_k]") + return gather_vqmm_m1_kernel_unchecked( + x, + codes, + scales, + codebook, + rhs_indices, + output_dims=output_dims, + code_bits=16, + rows_per_threadgroup=rows_per_threadgroup, + use_threadgroup_codebook=use_threadgroup_codebook, + use_decoded_codebook=False, + ) + + +def e8p_m1_per_route( + x, + codes, + scales, + codebook, + rhs_indices, + *, + output_dims: int, + rows_per_threadgroup: int = 32, + use_threadgroup_codebook: bool = True, +): + """Route ``r``'s activation row against expert ``rhs_indices[r]``: ``[routes, out]``.""" + + import mlx.core as mx + + from ramp.kernels.gather_vqmm import _qmv_output_dtype + + routes = int(rhs_indices.size) + if x.ndim != 2 or x.shape[0] != routes: + raise ValueError(f"per-route E8P M=1 needs x [{routes}, in], got {tuple(x.shape)}") + lanes_per_row, row_tiles = _launch_shape(output_dims, rows_per_threadgroup) + output_dtype = _qmv_output_dtype(x.dtype) + (out,) = _per_route_kernel()( + inputs=[mx.contiguous(x), codes, scales, codebook, rhs_indices.reshape(1, routes)], + template=[ + ("ROWS_PER_TG", rows_per_threadgroup), + ("LANES_PER_ROW", lanes_per_row), + ("OUT_T", output_dtype), + ("CODE_BITS", 16), + ("USE_THREADGROUP_CODEBOOK", int(use_threadgroup_codebook)), + ("USE_DECODED_CODEBOOK", 0), + ], + grid=(256, row_tiles, routes), + threadgroup=(256, 1, 1), + output_shapes=[(1, routes, output_dims)], + output_dtypes=[output_dtype], + ) + return out.reshape(routes, output_dims) diff --git a/src/ramp/nn/switch_linear.py b/src/ramp/nn/switch_linear.py index aadaf41..172407f 100644 --- a/src/ramp/nn/switch_linear.py +++ b/src/ramp/nn/switch_linear.py @@ -14,6 +14,7 @@ m1_use_decoded_codebook, m1_use_threadgroup_codebook, ) +from ramp.kernels.e8p_m1 import e8p_m1_per_route, e8p_m1_token_layout, should_use_e8p_m1 from ramp.ops.vq_switch import RouteStrategy from ramp.ops.vq_switch import gather_vqmm, vq_switch_qmv from keep.quant.rotation import apply_rotation_mx, apply_rotation_np, validate_rotation_matrix_mx @@ -332,7 +333,23 @@ def __call__(self, x: mx.array, indices: mx.array, *, sorted_indices: bool = Fal top_k = indices.shape[-1] flat_x = x.reshape((-1, self.input_dims)) flat_indices = indices.reshape((-1, top_k)) - if ( + if should_use_e8p_m1( + code_bits=self.code_bits, + tokens=flat_x.shape[0], + sorted_indices=sorted_indices, + route_strategy=self.route_strategy, + ): + # Opt-in (RAMP_E8P_M1=1): 16-bit codes at M=1 on GPUs without + # the NAX tensor-unit kernels, instead of the scalar fallback. + y = e8p_m1_token_layout( + flat_x, + self.codes, + scales, + self.codebook, + flat_indices, + output_dims=self.output_dims, + ) + elif ( flat_x.shape[0] == 1 and self.code_bits == 8 and not sorted_indices @@ -374,7 +391,21 @@ def __call__(self, x: mx.array, indices: mx.array, *, sorted_indices: bool = Fal route_shape = indices.shape flat_x = x.reshape((-1, self.input_dims)) flat_indices = indices.reshape((-1,)) - if ( + if indices.ndim >= 2 and should_use_e8p_m1( + code_bits=self.code_bits, + tokens=route_shape[0], + sorted_indices=sorted_indices, + route_strategy=self.route_strategy, + ): + y = e8p_m1_per_route( + flat_x, + self.codes, + scales, + self.codebook, + flat_indices, + output_dims=self.output_dims, + ).reshape((*route_shape, self.output_dims)) + elif ( route_shape[0] == 1 and flat_x.shape[0] <= 8 and self.code_bits == 8 diff --git a/tests/test_e8p_m1.py b/tests/test_e8p_m1.py new file mode 100644 index 0000000..b5a6622 --- /dev/null +++ b/tests/test_e8p_m1.py @@ -0,0 +1,145 @@ +"""Opt-in M=1 kernels for 16-bit E8P codes on GPUs without the NAX kernels.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +mx = pytest.importorskip("mlx.core") + +import ramp.nn.switch_linear as switch_linear # noqa: E402 +from ramp.kernels import e8p_m1 # noqa: E402 +from ramp.nn.switch_linear import QuantizedVQSwitchLinear # noqa: E402 + + +def _metal_available() -> bool: + metal = getattr(mx, "metal", None) + return bool(metal is not None and metal.is_available()) + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({}, True), + ({"code_bits": 8}, False), + ({"tokens": 2}, False), + ({"sorted_indices": True}, False), + ({"route_strategy": "sorted_tiled"}, False), + ({"route_strategy": "direct"}, True), + ], +) +def test_decision_table_with_the_flag_on(monkeypatch, kwargs, expected): + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "1") + args = {"code_bits": 16, "tokens": 1, "sorted_indices": False, "route_strategy": "auto", **kwargs} + assert e8p_m1.should_use_e8p_m1(**args) is expected + + +def test_flag_is_off_by_default_and_validated(monkeypatch): + monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) + assert e8p_m1.should_use_e8p_m1(code_bits=16, tokens=1, sorted_indices=False, route_strategy="auto") is False + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "maybe") + with pytest.raises(ValueError, match=e8p_m1.E8P_M1_ENV): + e8p_m1.e8p_m1_enabled() + + +def test_per_route_source_only_offsets_the_activation_reads(): + original = (Path(e8p_m1.__file__).parent / "gather_vqmm_m1.metal").read_text() + rewritten = e8p_m1.per_route_x_source() + assert rewritten.count("route * (codewords * 8u)") == 2 + # Nothing else changed: undoing the two rewrites restores the file exactly. + restored = rewritten + for old, new in e8p_m1._X_INDEX_REWRITES: + restored = restored.replace(new, old) + assert restored == original + + +def _layer(code_bits=16, experts=4, out=32, inp=64, seed=0): + rng = np.random.default_rng(seed) + weight = mx.array(rng.normal(size=(experts, out, inp)).astype(np.float32)) + return QuantizedVQSwitchLinear.from_weights(weight, group_size=32, code_bits=code_bits) + + +def test_switch_linear_routes_m1_16bit_calls_to_the_new_kernels(monkeypatch): + calls = [] + + def fake_token(x, codes, scales, codebook, rhs, *, output_dims): + calls.append(("token", tuple(x.shape), tuple(rhs.shape))) + return mx.zeros((1, rhs.shape[1], output_dims)) + + def fake_route(x, codes, scales, codebook, rhs, *, output_dims): + calls.append(("route", tuple(x.shape), tuple(rhs.shape))) + return mx.zeros((rhs.size, output_dims)) + + monkeypatch.setattr(switch_linear, "e8p_m1_token_layout", fake_token) + monkeypatch.setattr(switch_linear, "e8p_m1_per_route", fake_route) + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "1") + layer = _layer() + indices = mx.array([[0, 2, 3]], dtype=mx.uint32) + y = layer(mx.zeros((1, 64)), indices) # gate/up: one row, three experts + assert y.shape == (1, 3, 32) + y = layer(mx.zeros((1, 3, 64)), indices) # down: one row per route + assert y.shape == (1, 3, 32) + assert calls == [("token", (1, 64), (1, 3)), ("route", (3, 64), (3,))] + + +def test_switch_linear_leaves_8bit_and_flag_off_calls_alone(monkeypatch): + def boom(*args, **kwargs): # pragma: no cover - must not be called + raise AssertionError("opt-in kernel used without the flag or for 8-bit codes") + + monkeypatch.setattr(switch_linear, "e8p_m1_token_layout", boom) + monkeypatch.setattr(switch_linear, "e8p_m1_per_route", boom) + monkeypatch.setattr(switch_linear, "gather_vqmm", lambda *a, **k: mx.zeros((3, 32))) + monkeypatch.setattr(switch_linear, "gather_vqmm_m1_kernel", lambda *a, **k: mx.zeros((1, 3, 32))) + indices = mx.array([[0, 2, 3]], dtype=mx.uint32) + monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) + _layer(code_bits=16)(mx.zeros((1, 64)), indices) + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "1") + _layer(code_bits=8)(mx.zeros((1, 64)), indices) + + +def _dense(layer) -> np.ndarray: + from keep.vq.e8 import decode_weight_matrix + + codes = np.array(layer.codes) + scales = np.array(layer.effective_scales().astype(mx.float32)) + return np.stack( + [decode_weight_matrix(codes[e], scales[e], code_bits=16) for e in range(codes.shape[0])] + ) + + +@pytest.mark.skipif(not _metal_available(), reason="needs a Metal GPU") +@pytest.mark.parametrize("rows_per_threadgroup", [8, 32]) +def test_token_and_per_route_kernels_match_the_dense_decode(rows_per_threadgroup): + layer = _layer(experts=6, out=96, inp=128, seed=3) + dense = _dense(layer) + rng = np.random.default_rng(4) + rhs = mx.array([[5, 0, 3]], dtype=mx.uint32) + + x = rng.normal(size=(1, 128)).astype(np.float32) + got = e8p_m1.e8p_m1_token_layout( + mx.array(x), layer.codes, layer.effective_scales(), layer.codebook, rhs, + output_dims=96, rows_per_threadgroup=rows_per_threadgroup, + ) + want = np.stack([x[0] @ dense[e].T for e in (5, 0, 3)])[None] + np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) + + xr = rng.normal(size=(3, 128)).astype(np.float32) + got = e8p_m1.e8p_m1_per_route( + mx.array(xr), layer.codes, layer.effective_scales(), layer.codebook, rhs.reshape(-1), + output_dims=96, rows_per_threadgroup=rows_per_threadgroup, + ) + want = np.stack([xr[r] @ dense[e].T for r, e in enumerate((5, 0, 3))]) + np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) + + +def test_benchmark_synthetic_layers_match_the_shipped_byte_accounting(): + from benchmarks import bench_e8p_m1 as bench + + gate = bench.synthetic_projection(experts=2, out=2048, inp=4096, group_size=512, seed=0) + down = bench.synthetic_projection(experts=2, out=4096, inp=2048, group_size=512, seed=0) + # The roadmap's V4-Flash decode table: 2,129,920 B per projection per expert. + assert bench.projection_bytes_per_expert(gate) == 2_129_920 + assert bench.projection_bytes_per_expert(down) == 2_129_920 + assert gate.code_bits == 16 and gate.codes.dtype == mx.uint16 From 7deda771be1e417f32c5fc9c590ebad4090382ad Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:25:06 +0000 Subject: [PATCH 07/74] docs: Phase 0 kill-test runbook and one-command Mac runner scripts/phase0_kill_tests.sh runs every Phase 0 kill test that needs no model download (mxfp8 residents, fused sinkhorn, single-token E8P, and the Metal parity tests). It refuses to start with custom wired limits set, writes JSON evidence under artifacts/benchmarks/phase0--/ and prints one verdict per test. If KEEP_RESIDENT_DIR and KEEP_VQ_DIR are set, it also runs the real-weight variants. docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md lists each opt-in switch, its kill criterion and the decision a pass unlocks. It also gives the calibration steps the REAP and residency tests need, which must write into a new output directory because the runner skips existing sessions. --- .../2026-11-05-phase0-kill-tests.md | 70 +++++++++++++++++++ scripts/phase0_kill_tests.sh | 70 +++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md create mode 100755 scripts/phase0_kill_tests.sh diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md new file mode 100644 index 0000000..44f303c --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -0,0 +1,70 @@ +# Phase 0 kill tests: toward DeepSeek-V4.1-Flash on one 128 GB Mac + +Date: 2026-11-05. Target machine: the M4 Max (128 GB, 409.4 GB/s measured +MLX read). Phase 0 is the work that needs no V4.1 download and can be proven +on the V4-Flash artifact. Every runtime change below is **opt-in**; defaults +are unchanged until the kill test that guards it passes on the target Mac. + +Run everything that needs no model download in one go: + +```bash +scripts/phase0_kill_tests.sh +``` + +It writes JSON evidence to `artifacts/benchmarks/phase0--/`. That +path is gitignored, so force-add it (`git add -f`). + +## What each test decides + +| # | Change | Opt-in switch | Kill test | KILL when | If it passes | +|---|---|---|---|---|---| +| 1 | FP8 residents kept as source bytes (weights-only mxfp8, incl. `wo_a`) | `resident_format="mxfp8"`; `--resident-format mxfp8` on the headline harness | `benchmarks/bench_dsv4_mxfp8_residents.py` | any weight differs from the bf16 decode; mxfp8 < 1.5x bf16 at M=1; > 1.1x affine-8's time | bind the composite with mxfp8 and rerun the headline series | +| 3 | Single-token E8P kernels for 16-bit codes on GPUs without NAX | `RAMP_E8P_M1=1` | `benchmarks/bench_e8p_m1.py` | outputs disagree, or < 100 GB/s effective on one routed FFN | >= 200 GB/s: make it the M4 default. 100-200: decode each codeword once, then re-measure | +| 5 | Headline rows also gate memory coming back in (schema v2) | always on for new rows; v1 rows keep their rule | the next headline series | — | quote only v2 rows | +| 6 | One-dispatch hyper-connection sinkhorn | `RAMP_HC_SINKHORN=metal` | `benchmarks/bench_hc_sinkhorn.py` | < 3x faster at decode shape, or > 2e-6 relative difference | set it for the headline | +| R3 | REAP saliency from the real gate, in the calibration pass | always collected in `calibration` mode | `produce_dsv4_teacher_cache.py reap-select --keep 128` on V4-Flash stats | median pruned-set Jaccard vs route frequency >= 0.9 (REAP adds nothing) | use REAP order for the prune and the hot set | +| 7 | Expert hot set in RAM, rest on SSD | none (simulation only) | `benchmarks/replay_dsv4_expert_residency.py` | best fetching policy misses > 5% of routed slots at the hot-set size you'd ship | worth building the SSD tier | + +Tests R3 and 7 need a calibration run, because the per-session REAP statistics +and recorded routes are new. Rerun the V4-Flash calibration split (5.11 h last +time) into a **new** output directory. The runner resumes by skipping sessions +that already exist, so the old directory would keep sessions without the new +arrays, and `finalize-reap` refuses those. Then: + +```bash +uv run python benchmarks/produce_dsv4_teacher_cache.py finalize-reap --out-dir ~/keep-artifacts/dsv4-teacher-calibration-reap +``` + +```bash +uv run python benchmarks/produce_dsv4_teacher_cache.py reap-select --stats ~/keep-artifacts/dsv4-teacher-calibration-reap/reap-stats.npz --keep 128 +``` + +```bash +uv run python benchmarks/replay_dsv4_expert_residency.py --calibration-dir ~/keep-artifacts/dsv4-teacher-calibration-reap --reap-stats ~/keep-artifacts/dsv4-teacher-calibration-reap/reap-stats.npz --hot 96,128,160 --expert-mb 6.39 +``` + +The replay's default `--expert-mb` is V4.1's 8.99 MB. A V4-Flash expert is +6.39 MB (3 x 2,129,920 B). + +## What was verified where + +On Linux with CPU MLX 0.31.2: + +- the mxfp8 byte copy dequantizes bit-identically at all eight V4-Flash + resident shapes, and `mx.quantize(mode="mxfp8")` does not (608 of 98,304 + values changed); +- REAP statistics match a float64 reference, and slicing the router equals + masking it; +- the residency replay reproduces hand-counted LRU and decayed-count misses; +- the sinkhorn reference reproduces the model's compiled path. + +Everything Metal (kernel parity, every speed number) is untested until the +script above runs on the Mac. + +## Known gaps + +- `benchmarks/bench_dsv4_mtp_headline.py` validates speculative rows against + `EXPECTED_VERIFY_IMPLEMENTATION = "nax_e8p_m32n64"`, so MTP headline rows + cannot validate on an M4 until the verify window has a non-NAX E8P kernel. +- The E8P M=1 branch decodes each codeword once per dimension. Decoding once + per codeword is the first optimisation if the kill test lands in TUNE. diff --git a/scripts/phase0_kill_tests.sh b/scripts/phase0_kill_tests.sh new file mode 100755 index 0000000..d75f554 --- /dev/null +++ b/scripts/phase0_kill_tests.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Phase 0 kill tests that need no model download. Run on the target Mac: +# +# scripts/phase0_kill_tests.sh # writes artifacts/benchmarks/phase0--/ +# +# Each benchmark exits 1 on KILL. The script runs them all, then prints one +# verdict line per test. Optional real-weight variants, if the files exist: +# KEEP_RESIDENT_DIR=~/keep-artifacts/dsv4-residents-release-precision-20260819 +# KEEP_VQ_DIR=~/keep-artifacts/dsv4-vq-e8p-g512 +set -uo pipefail + +for name in GLM_MLX_WIRED_LIMIT_GB GLM_SINGLE_HOST_MLX_WIRED_LIMIT_GB; do + if [[ -n "${!name:-}" ]]; then + echo "unset $name first: kill tests run under the default wired-memory limits" >&2 + exit 2 + fi +done + +repo="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo" +out="artifacts/benchmarks/phase0-$(hostname -s)-$(date +%Y%m%d-%H%M%S)" +mkdir -p "$out" +status=0 + +run() { + local label="$1"; shift + echo "== $label" + if uv run python "$@"; then echo "$label: ok"; else echo "$label: KILL or error (exit $?)"; status=1; fi +} + +run "mxfp8 residents (synthetic V4-Flash shapes)" \ + benchmarks/bench_dsv4_mxfp8_residents.py --synthetic --output-json "$out/mxfp8-residents-synthetic.json" +if [[ -n "${KEEP_RESIDENT_DIR:-}" && -d "${KEEP_RESIDENT_DIR}" ]]; then + run "mxfp8 residents (real layer 20)" \ + benchmarks/bench_dsv4_mxfp8_residents.py --checkpoint "$KEEP_RESIDENT_DIR" --layer 20 \ + --output-json "$out/mxfp8-residents-layer20.json" +fi +run "fused hyper-connection sinkhorn" \ + benchmarks/bench_hc_sinkhorn.py --output-json "$out/hc-sinkhorn.json" +run "E8P single-token kernels (synthetic)" \ + benchmarks/bench_e8p_m1.py --output-json "$out/e8p-m1-synthetic.json" +if [[ -n "${KEEP_VQ_DIR:-}" && -d "${KEEP_VQ_DIR}" ]]; then + run "E8P single-token kernels (real layer 20)" \ + benchmarks/bench_e8p_m1.py --artifact-dir "$KEEP_VQ_DIR" --layer 20 \ + --output-json "$out/e8p-m1-layer20.json" +fi + +echo "== Metal parity tests" +if uv run --group dev python -m pytest -q \ + tests/test_hc_sinkhorn.py tests/test_e8p_m1.py tests/test_fp8_block_dequant.py \ + tests/test_bench_dsv4_mxfp8_residents.py "tests/test_deepseek_v4_flash_adapter.py" -k "mxfp8 or sinkhorn or e8p or fp8" \ + | tee "$out/metal-parity-tests.txt"; then + echo "Metal parity tests: ok" +else + echo "Metal parity tests: FAILED"; status=1 +fi + +echo +echo "Verdicts:" +for f in "$out"/*.json; do + uv run python - "$f" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +v = d.get("verdict") +v = v.get("verdict") if isinstance(v, dict) else v +print(f" {sys.argv[1].rsplit('/', 1)[-1]}: {v}") +PY +done +echo "Evidence: $out (force-add it: git add -f $out)" +exit $status From b392eb3165419e29e314d3463274baa49279077e Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:29:48 +0000 Subject: [PATCH 08/74] docs: V4.1 plan and Phase 0 closeout status Records the ranked plan for running DeepSeek-V4.1-Flash on one 128 GB Mac, what Phase 0 built and verified on this branch, the next actions in order (Mac kill tests, REAP calibration rerun, CI fix), and the open decisions. --- .../2026-11-05-v41-plan-and-phase0-status.md | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md new file mode 100644 index 0000000..e433c8a --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -0,0 +1,95 @@ +# DeepSeek-V4.1-Flash on one 128 GB Mac: plan and Phase 0 status + +Date: 2026-11-05. Branch `jack/hopeful-cori-iszsfq`, draft PR jackmaxwil/keep#3. +Target machine: M4 Max, 128 GB, 409.4 GB/s measured MLX read. + +## The plan in brief + +This synthesises three earlier analyses (oMLX's oQ quantization, oMLX's V4.1 +Engram/offload work, and REAP pruning) and checks them against this repo. + +- **Memory and speed come from different weights.** Each routed expert kept in + every layer costs 0.359 GB of RAM (V4.1, 2.031 bpw) but adds no per-token + reads. Resident weights and kernels set decode speed. So: prune to fit, fix + residents and kernels to go fast, and give MTP whatever memory is left. +- **Speed is a kernel problem first.** The measured V4-Flash decode floor is + 270–390 ms/token (roadmap). On an M4 Max, 16-bit E8P experts ran on the + scalar kernel, because NAX is M5-only and the M=1 fast paths were 8-bit-only. +- **Fit forces a ~50% prune**, about 176–192 of 384 experts per layer. That is + where REAP's published results lose most outside code (Kimi-K2 at 50%: −17.6% + multiple choice, −54% multi-turn tool use). KEEP's quality gate is in-domain, + so an out-of-domain teacher split must decide the expert count. +- **Plausible end state (estimate):** ~31 tok/s autoregressive and 37–53 with + MTP, at 81–87 GB in flight. Engram's 203 GB stays on SSD. + +### The top 20, by execution order + +Required for V4.1 to exist at all: +- **R1:** port V4.1 with reference parity. +- **R2:** read Engram from SSD. +- **R3:** REAP done right (real-gate saliency plus router slicing). + +Ranked levers: +1. FP8 residents as mxfp8 byte copies +2. Out-of-domain teacher split to choose the calibration mix and N +3. Non-NAX 16-bit E8P kernels +4. Standard benchmarks compared with oMLX and GGUF builds +5. Quiet-window harness that gates on inbound paging +6. Decode-floor fusion (sinkhorn, descriptors, compile) +7. REAP-ordered fast/exact residency (SSD tier) +8. MTP stack +9. Recovery distillation +10. Ship V4-Flash first +11. Persistent prefix-KV cache +12. Prune-vs-bits frontier +13. Resident trims (head and embedding to 8 bits) +14. Two-Mac RDMA calibration +15. 4-bit attention "turbo" +16. 4-bit Engram tables +17. Fit polish + +## Phase 0: done in this branch (no V4.1 download needed) + +| Plan item | Commit | Status | +|---|---|---| +| 1 mxfp8 residents (incl. `wo_a`) | `3c6d9f1` | Built, opt-in. Bit-exact on CPU at all 8 V4-Flash shapes. **Speed untested** | +| 5 inbound-paging gate (row schema v2) | `450c925` | Built. Unit-tested | +| R3 REAP saliency + router slicing | `bd7fd54` | Built. Matches a float64 reference; slicing equals masking | +| 7 residency replay (SSD-tier kill test) | `33a5681` | Built. Hand-checked miss counts | +| 6 fused sinkhorn | `5baed30` | Built, opt-in. **Metal untested** | +| 3 single-token E8P on M1–M4 | `00580c4` | Built, opt-in. **Metal untested** | +| Kill-test runner and runbook | `7deda77` | `scripts/phase0_kill_tests.sh`, `2026-11-05-phase0-kill-tests.md` | + +Regression check (Linux, CPU MLX): on the 33 test files that import changed +modules, the failure set is identical to `main`'s (308 failures needing Metal +or Mac-local files), plus 22 newly passing tests. + +## Next actions, in order + +1. On the M4 Max, run `scripts/phase0_kill_tests.sh`. Each test is PASS, KILL + or TUNE per the runbook. +2. Rerun V4-Flash calibration into a **new** directory, then run + `finalize-reap`, `reap-select --keep 128` and the residency replay. +3. If mxfp8 passes, bind the composite with `--resident-format mxfp8` and rerun + the headline series. Quote v2 rows only. +4. Fix CI. `validate-recipes` fails on `main` because strict recipes refuse + `--no-hash`, so the pytest step never runs. The proposed workflow patch is + in the PR description. +5. Then Phase 1 (V4.1 port, Engram, two-Mac calibration), which waits on your + decisions below. + +## Open decisions (yours) + +- **Serving machine.** M4 Max (this plan) or M5 Max (decision D2, 2026-10-06). +- **Scope.** Coding-agent specialist or generalist. This sets the calibration + mix and the prune depth. +- **Disk and download.** The V4.1 source is 510 GB; the M4 had 73 GiB free on + Oct 6. +- **Exact mode.** In v1 or later. It adds 69 GB to the download. + +## Known gaps + +- MTP headline rows require the M5-only `nax_e8p_m32n64`, so they won't + validate on an M4. +- The E8P M=1 branch decodes each codeword once per dimension. Decoding once + per codeword is the next optimisation if its verdict is TUNE. From 8618f29c3a51d46c4399194b10af81607ce1c824 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:26:56 -0800 Subject: [PATCH 09/74] bench: Phase 0 kill-test evidence from the M4 Max mxfp8 residents: exact at all eight shapes, but KILL on speed. At M=1 it runs 0.88x bf16 and 1.31x affine-8 (three runs: 0.88-0.90x bf16). E8P M=1: outputs agree, KILL at 52.8 GB/s best (2.38x production, rows_per_threadgroup 32). Sinkhorn: PASS at decode (3.69x, 5.7e-7), but 0.50x at 1024 rows. Also fix test_hyper_connection_output_with_the_flag, which skipped on Linux and failed on Metal: one-layer args need num_hash_layers=0. --- .../e8p-m1-synthetic.json | 62 +++ .../hc-sinkhorn.json | 28 ++ .../metal-parity-tests-after-fix.txt | 11 + .../metal-parity-tests.txt | 64 +++ .../mxfp8-residents-synthetic.json | 431 ++++++++++++++++++ tests/test_hc_sinkhorn.py | 2 +- 6 files changed, 597 insertions(+), 1 deletion(-) create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic.json create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/hc-sinkhorn.json create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests-after-fix.txt create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests.txt create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic.json diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic.json new file mode 100644 index 0000000..6715e39 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic.json @@ -0,0 +1,62 @@ +{ + "best": { + "effective_gb_per_s": 52.75492459836312, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.7267295004567131, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.3768878351187355, + "threadgroup_codebook": true + }, + "configs": [ + { + "effective_gb_per_s": 26.706688923324624, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 1.4355414896272123, + "rows_per_threadgroup": 8, + "speedup_over_production": 1.2032773148939389, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 37.284508678850266, + "max_relative_diff_vs_production": 0.000638977624475956, + "median_ms_per_ffn": 1.0282704897690564, + "rows_per_threadgroup": 16, + "speedup_over_production": 1.6798639329282228, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 52.75492459836312, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.7267295004567131, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.3768878351187355, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 44.31135339640817, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.8652085089124739, + "rows_per_threadgroup": 64, + "speedup_over_production": 1.9964603806644017, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 45.07216109711928, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.8506039885105565, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.0307387837225495, + "threadgroup_codebook": false + } + ], + "device": "Device(gpu, 0)", + "ffn_bytes": 38338560, + "mlx_version": "0.31.2", + "outputs_agree": true, + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "production_effective_gb_per_s": 22.194957548650077, + "production_median_ms_per_ffn": 1.7273545090574771, + "record_type": "ramp_e8p_m1_kill_test_v1", + "source": "synthetic", + "verdict": "KILL" +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/hc-sinkhorn.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/hc-sinkhorn.json new file mode 100644 index 0000000..e686b81 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/hc-sinkhorn.json @@ -0,0 +1,28 @@ +{ + "device": "Device(gpu, 0)", + "mlx_version": "0.31.2", + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "reasons": [], + "record_type": "ramp_hc_sinkhorn_kill_test_v1", + "rows": [ + { + "compiled_ms_per_call": 1.407559666025918, + "compiled_ms_per_decode_step": 121.05013127822895, + "fused_ms_per_call": 0.3815831679967232, + "fused_ms_per_decode_step": 32.816152447718196, + "max_relative_diff": 5.69557300877932e-07, + "rows": 1, + "speedup": 3.688736254839221 + }, + { + "compiled_ms_per_call": 0.9860470839776099, + "compiled_ms_per_decode_step": 84.80004922207445, + "fused_ms_per_call": 1.9785255839815363, + "fused_ms_per_decode_step": 170.15320022241212, + "max_relative_diff": 1.4791691000937135e-06, + "rows": 1024, + "speedup": 0.4983746947529043 + } + ], + "verdict": "PASS" +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests-after-fix.txt b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests-after-fix.txt new file mode 100644 index 0000000..d6f6f3b --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests-after-fix.txt @@ -0,0 +1,11 @@ +...................................................................... [100%] +=============================== warnings summary =============================== +:491 + :491: DeprecationWarning: builtin type SwigPyPacked has no __module__ attribute + +:491 + :491: DeprecationWarning: builtin type SwigPyObject has no __module__ attribute + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +70 passed, 80 deselected, 2 warnings in 5.96s +:0: DeprecationWarning: builtin type swigvarlink has no __module__ attribute diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests.txt b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests.txt new file mode 100644 index 0000000..3fb8e66 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/metal-parity-tests.txt @@ -0,0 +1,64 @@ +........F............................................................. [100%] +=================================== FAILURES =================================== +__________________ test_hyper_connection_output_with_the_flag __________________ + +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x119e616e0> + + @pytest.mark.skipif(not _metal_available(), reason="needs a Metal GPU") + def test_hyper_connection_output_with_the_flag(monkeypatch): + from ramp.models.deepseek_v4_flash_adapter import ( + DeepseekV4FlashVQModelArgs, + HyperConnection, + ) + +> args = DeepseekV4FlashVQModelArgs(hidden_size=64, hc_mult=HC, num_hidden_layers=1, compress_ratios=[0]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/test_hc_sinkhorn.py:88: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +:48: in __init__ + ??? +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = DeepseekV4FlashVQModelArgs(model_type='deepseek_v4', vocab_size=129280, hidden_size=64, intermediate_size=18432, moe_i... dspark_target_layer_ids=[], dspark_markov_rank=256, n_mtp_layers=0, tie_word_embeddings=False, topk_method='noaux_tc') + + def __post_init__(self) -> None: + if not self.compress_ratios: + n = self.num_hidden_layers + self.compress_ratios = ( + [0] + + [4 if i % 2 else 128 for i in range(max(n - 2, 0))] + + ([0] if n >= 2 else []) + ) + # The drafter ratios live past ``num_hidden_layers``; keep them so the + # MTP blocks can be built from the same list, then truncate the + # backbone view exactly as upstream does. + self.mtp_compress_ratios = list(self.compress_ratios[self.num_hidden_layers :]) + self.compress_ratios = list(self.compress_ratios[: self.num_hidden_layers]) + if len(self.compress_ratios) != self.num_hidden_layers: + raise ValueError( + "`compress_ratios` must have one entry per hidden layer, " + f"got {len(self.compress_ratios)} for {self.num_hidden_layers} layers." + ) + bad = [r for r in self.compress_ratios if r not in (0, 4, 128)] + if bad: + raise ValueError(f"Unsupported DeepSeek-V4 compress ratios: {bad}") + if self.num_hash_layers > self.num_hidden_layers: +> raise ValueError( + "`num_hash_layers` cannot exceed `num_hidden_layers`: " + f"{self.num_hash_layers} > {self.num_hidden_layers}" + ) +E ValueError: `num_hash_layers` cannot exceed `num_hidden_layers`: 3 > 1 + +src/ramp/models/deepseek_v4_flash_adapter.py:300: ValueError +=============================== warnings summary =============================== +:491 + :491: DeprecationWarning: builtin type SwigPyPacked has no __module__ attribute + +:491 + :491: DeprecationWarning: builtin type SwigPyObject has no __module__ attribute + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +=========================== short test summary info ============================ +FAILED tests/test_hc_sinkhorn.py::test_hyper_connection_output_with_the_flag +1 failed, 69 passed, 80 deselected, 2 warnings in 11.74s diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic.json new file mode 100644 index 0000000..9928525 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic.json @@ -0,0 +1,431 @@ +{ + "created_utc": "2026-11-06T03:25:00+00:00", + "device": "Device(gpu, 0)", + "device_info": { + "architecture": "applegpu_g16s", + "device_name": "Apple M4 Max", + "max_buffer_length": 86586540032, + "max_recommended_working_set_size": 115448725504, + "memory_size": 137438953472, + "resource_limit": 499000 + }, + "exactness": { + "attn.wkv": { + "bf16_bytes": 4194304, + "bit_mismatches": 0, + "elements": 2097152, + "mxfp8_bytes": 2162688 + }, + "attn.wo_a": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "attn.wo_b": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "attn.wq_a": { + "bf16_bytes": 8388608, + "bit_mismatches": 0, + "elements": 4194304, + "mxfp8_bytes": 4325376 + }, + "attn.wq_b": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "ffn.shared_experts.w1": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + }, + "ffn.shared_experts.w2": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + }, + "ffn.shared_experts.w3": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + } + }, + "layer": null, + "mlx_version": "0.31.2", + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "record_type": "dsv4_mxfp8_resident_kill_test_v1", + "settings": { + "copies": 8, + "iterations": 200, + "rounds": 5, + "seed": 20261105, + "warmup": 20 + }, + "shapes": { + "attn.wkv": [ + 512, + 4096 + ], + "attn.wo_a": [ + 8192, + 4096 + ], + "attn.wo_b": [ + 4096, + 8192 + ], + "attn.wq_a": [ + 1024, + 4096 + ], + "attn.wq_b": [ + 32768, + 1024 + ], + "ffn.shared_experts.w1": [ + 2048, + 4096 + ], + "ffn.shared_experts.w2": [ + 4096, + 2048 + ], + "ffn.shared_experts.w3": [ + 2048, + 4096 + ] + }, + "source": "synthetic", + "timings": { + "attn.wkv": { + "affine8_g64": { + "effective_gb_per_s": 12.346655118439584, + "median_ms_per_call": 0.18047187506454065, + "round_ms": [ + 0.1531339599750936, + 0.14204958497430198, + 0.18047187506454065, + 0.445007084927056, + 0.3591016650898382 + ], + "weight_bytes": 2228224 + }, + "bf16": { + "effective_gb_per_s": 12.865132878226667, + "median_ms_per_call": 0.3260210399457719, + "round_ms": [ + 0.17775187487131916, + 0.16968333497061394, + 0.3260210399457719, + 0.33897583503858186, + 0.4410322899639141 + ], + "weight_bytes": 4194304 + }, + "mxfp8": { + "effective_gb_per_s": 7.474577455789129, + "median_ms_per_call": 0.2893391650286503, + "round_ms": [ + 0.18327979007153772, + 0.16327021003235132, + 0.2893391650286503, + 0.3063174999260809, + 0.3682972899696324 + ], + "weight_bytes": 2162688 + } + }, + "attn.wo_a": { + "affine8_g64": { + "effective_gb_per_s": 102.68116105737253, + "median_ms_per_call": 0.34720666997600347, + "round_ms": [ + 0.24603687488706782, + 0.3152260449132882, + 0.34720666997600347, + 0.4145656249602325, + 0.49060249992180616 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 181.9718413458899, + "median_ms_per_call": 0.36878707993309945, + "round_ms": [ + 0.259654375113314, + 0.3418068749306258, + 0.39245832987944596, + 0.36878707993309945, + 0.37461583502590656 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 75.11911070382045, + "median_ms_per_call": 0.4606418749608565, + "round_ms": [ + 0.22329687504679896, + 0.4196122899884358, + 0.4606418749608565, + 0.49901457998203114, + 0.6001195800490677 + ], + "weight_bytes": 34603008 + } + }, + "attn.wo_b": { + "affine8_g64": { + "effective_gb_per_s": 94.23450790150733, + "median_ms_per_call": 0.3783283299708273, + "round_ms": [ + 0.356820000015432, + 0.5951014599122573, + 0.3783283299708273, + 0.5414641699462663, + 0.3499670849123504 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 124.70975174653974, + "median_ms_per_call": 0.5381204200966749, + "round_ms": [ + 0.3941564599517733, + 0.46919395492295735, + 0.5381204200966749, + 0.5693583299580496, + 0.6413285399321467 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 72.3405002191153, + "median_ms_per_call": 0.4783352049707901, + "round_ms": [ + 0.4622772899165284, + 0.4783352049707901, + 0.6258887500735, + 0.7353650000004563, + 0.471238544996595 + ], + "weight_bytes": 34603008 + } + }, + "attn.wq_a": { + "affine8_g64": { + "effective_gb_per_s": 25.627142711392384, + "median_ms_per_call": 0.17389562504831702, + "round_ms": [ + 0.17389562504831702, + 0.16125479000038467, + 0.21068166504846886, + 0.16948375006904826, + 0.1889118750113994 + ], + "weight_bytes": 4456448 + }, + "bf16": { + "effective_gb_per_s": 40.897470608003545, + "median_ms_per_call": 0.20511312497546896, + "round_ms": [ + 0.23084375003236346, + 0.20371875012642704, + 0.20511312497546896, + 0.20693083002697676, + 0.19877270999131724 + ], + "weight_bytes": 8388608 + }, + "mxfp8": { + "effective_gb_per_s": 20.422224049928882, + "median_ms_per_call": 0.21179750008741394, + "round_ms": [ + 0.2476037500309758, + 0.1843235400156118, + 0.2059089599060826, + 0.2374814600625541, + 0.21179750008741394 + ], + "weight_bytes": 4325376 + } + }, + "attn.wq_b": { + "affine8_g64": { + "effective_gb_per_s": 117.4911368600051, + "median_ms_per_call": 0.3034406249935273, + "round_ms": [ + 0.3034406249935273, + 0.29054125014226884, + 0.31788437510840595, + 0.2908995850884821, + 0.3122981249180157 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 213.11194810661834, + "median_ms_per_call": 0.3148995849187486, + "round_ms": [ + 0.3262793750036508, + 0.3105731248797383, + 0.32184896001126617, + 0.29918750005890615, + 0.3148995849187486 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 86.6203516645367, + "median_ms_per_call": 0.3994789600255899, + "round_ms": [ + 0.4192366650386248, + 0.3994789600255899, + 0.39770604504155926, + 0.39270124994800426, + 0.4122983349952847 + ], + "weight_bytes": 34603008 + } + }, + "ffn.shared_experts.w1": { + "affine8_g64": { + "effective_gb_per_s": 18.53428043671034, + "median_ms_per_call": 0.4808870800479781, + "round_ms": [ + 0.5062785399786662, + 0.4808870800479781, + 0.5387225000595208, + 0.4101327050011605, + 0.3056120849214494 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 34.19429269589055, + "median_ms_per_call": 0.49064375009038486, + "round_ms": [ + 0.29815021000104025, + 0.5594412499340251, + 0.5425206250220072, + 0.4029964600340463, + 0.49064375009038486 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 20.047966791213195, + "median_ms_per_call": 0.43150270998012275, + "round_ms": [ + 0.23859770997660235, + 0.4383347899420187, + 0.43150270998012275, + 0.4810493749391753, + 0.4037372949824203 + ], + "weight_bytes": 8650752 + } + }, + "ffn.shared_experts.w2": { + "affine8_g64": { + "effective_gb_per_s": 34.47281436689568, + "median_ms_per_call": 0.25854854509816505, + "round_ms": [ + 0.2853339599096216, + 0.25854854509816505, + 0.34017124999081716, + 0.22430207987781614, + 0.18124374997569248 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 43.488465694280784, + "median_ms_per_call": 0.3857854199304711, + "round_ms": [ + 0.6273304150090553, + 0.3663843750837259, + 0.3857854199304711, + 0.27615958504611626, + 0.45533145996159874 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 25.411608398029266, + "median_ms_per_call": 0.34042520506773144, + "round_ms": [ + 0.3799189599521924, + 0.36401020508492365, + 0.34042520506773144, + 0.22804603999247774, + 0.32679124997230247 + ], + "weight_bytes": 8650752 + } + }, + "ffn.shared_experts.w3": { + "affine8_g64": { + "effective_gb_per_s": 17.94197381660171, + "median_ms_per_call": 0.49676228998578154, + "round_ms": [ + 0.49676228998578154, + 0.45178312502685003, + 0.3772306250175461, + 0.6179714600148145, + 0.6743810400075745 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 42.092094426890746, + "median_ms_per_call": 0.39858353993622586, + "round_ms": [ + 0.353656039951602, + 0.5094829149311408, + 0.5573883350007236, + 0.39488395501393825, + 0.39858353993622586 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 10.516395663417452, + "median_ms_per_call": 0.8225966649479233, + "round_ms": [ + 0.48426271008793265, + 0.4548260450246744, + 0.827316249924479, + 1.3013379200128838, + 0.8225966649479233 + ], + "weight_bytes": 8650752 + } + } + }, + "verdict": { + "bit_mismatches": 0, + "exact": true, + "mxfp8_speedup_over_bf16": 0.8817270082742493, + "mxfp8_time_over_affine8": 1.3109614365218596, + "reasons": [ + "mxfp8 is 0.882x bf16, below the 1.5x bar", + "mxfp8 takes 1.311x affine-8's time, above 1.1x" + ], + "speed_valid": true, + "summed_median_ms": { + "affine8_g64": 2.6195410401851404, + "bf16": 3.0279539598268457, + "mxfp8": 3.434117285069078 + }, + "verdict": "KILL" + } +} diff --git a/tests/test_hc_sinkhorn.py b/tests/test_hc_sinkhorn.py index d5ac829..4b30d19 100644 --- a/tests/test_hc_sinkhorn.py +++ b/tests/test_hc_sinkhorn.py @@ -85,7 +85,7 @@ def test_hyper_connection_output_with_the_flag(monkeypatch): HyperConnection, ) - args = DeepseekV4FlashVQModelArgs(hidden_size=64, hc_mult=HC, num_hidden_layers=1, compress_ratios=[0]) + args = DeepseekV4FlashVQModelArgs(hidden_size=64, hc_mult=HC, num_hidden_layers=1, num_hash_layers=0, compress_ratios=[0]) hc = HyperConnection(args) rng = np.random.default_rng(1) hc.fn = mx.array(rng.normal(size=hc.fn.shape).astype(np.float32) * 0.05) From 1052620db5893cdd40c7b17d9ee488850bcbcd28 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:40:25 -0800 Subject: [PATCH 10/74] docs: decoding each E8P codeword once is slower on the M4 Max A decode-once M=1 kernel (popcount parity, one abs-grid lookup per codeword, wide code loads) ran at 113 GB/s against 154 GB/s for the existing per-dimension branch, measured kernel-only with 256 calls per sync and best of 20 interleaved runs. The compiler already lifts the per-codeword work out of the 8-dimension loop. The kernel and both scripts are kept under the Phase 0 evidence directory; the source tree is unchanged. The kill test's KILL comes mostly from syncing after every FFN. With 43 dependent FFNs per sync, as in one decode step, RAMP_E8P_M1=1 reaches 118 GB/s (TUNE) against 50 GB/s for production. --- .../e8p-decode-once/chain.py | 29 +++++ .../e8p-decode-once/e8p_m1.decode-once.metal | 100 ++++++++++++++++++ .../e8p-decode-once/kbench.py | 38 +++++++ benchmarks/bench_e8p_m1.py | 2 +- .../2026-11-05-phase0-kill-tests.md | 11 +- .../2026-11-05-v41-plan-and-phase0-status.md | 9 +- src/ramp/kernels/e8p_m1.py | 6 +- 7 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/chain.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/e8p_m1.decode-once.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/kbench.py diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/chain.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/chain.py new file mode 100644 index 0000000..569887b --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/chain.py @@ -0,0 +1,29 @@ +import os, sys, time +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection, _ffn, projection_bytes_per_expert, PROJECTIONS +from ramp.kernels import e8p_m1 +layers = { + "gate_proj": synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1), + "up_proj": synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=2), + "down_proj": synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3), +} +rng = np.random.default_rng(0) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +ffn_bytes = 6 * sum(projection_bytes_per_expert(layers[p]) for p in PROJECTIONS) +x0 = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) + +def chain(n): + x = x0 + for i in range(n): # dependent, like consecutive layers in one decode step + y = _ffn(layers, x, routes[i % 64], 2048) + x = (x + y.sum(axis=0, keepdims=True) * 1e-3).astype(mx.float16) + return x + +for flag in ("0", "1"): + os.environ[e8p_m1.E8P_M1_ENV] = flag + for n in (1, 43): + for _ in range(3): mx.eval(chain(n)) + best = min((lambda a: (mx.eval(chain(n)), time.perf_counter() - a)[1])(time.perf_counter()) for _ in range(30)) + ms = best / n * 1e3 + print(f"E8P_M1={flag} ffns/eval={n:2d} {ms:.4f} ms/ffn {ffn_bytes / (ms / 1e3) / 1e9:.1f} GB/s") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/e8p_m1.decode-once.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/e8p_m1.decode-once.metal new file mode 100644 index 0000000..9e1aac9 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/e8p_m1.decode-once.metal @@ -0,0 +1,100 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes, decoding each +// codeword once into eight values (the generic M=1 kernel decodes it once per +// dimension, with an 8-step parity loop each time). +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble shuffle(d) = {0,4,1,5,2,6,3,7}[d], so nibbles 0-3 +// pair with the even activations and nibbles 4-7 with the odd ones. Bit p of +// the effective signs negates nibble p, and every value gets +-0.25 by parity. +// +// Each lane loads CHUNK (4 or 8) codewords at once; 2-byte loads leave the +// matvec latency-bound. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); + +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +uint row_slot = lane / uint(LANES_PER_ROW); +uint row_lane = lane - row_slot * uint(LANES_PER_ROW); +uint row = row_tile * uint(ROWS_PER_TG) + row_slot; + +threadgroup uint local_codebook[256]; +if (uint(USE_THREADGROUP_CODEBOOK) != 0u) { + for (uint idx = lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +float acc = 0.0f; +if (row_slot < uint(ROWS_PER_TG) && row < out_dim) { + uint code_base = (expert * out_dim + row) * codewords; + uint scale_base = (expert * out_dim + row) * groups; + // One wide load of CHUNK codewords per lane per step; the host checks + // that codewords per group is a multiple of CHUNK. + uint chunks = codewords / uint(CHUNK); + for (uint chunk = row_lane; chunk < chunks; chunk += uint(LANES_PER_ROW)) { + uint cw0 = chunk * uint(CHUNK); + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + code_base))[chunk]; + } else { + words.xy = ((const device uint2*)(codes + code_base))[chunk]; + } + float chunk_acc = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint abs_code = uint(USE_THREADGROUP_CODEBOOK) != 0u + ? local_codebook[codeword >> 8u] + : codebook[codeword >> 8u]; + float4 v_lo = (float4((uint4(abs_code) >> uint4(0u, 4u, 8u, 12u)) & 0xFu) - 8.0f) * 0.5f; + float4 v_hi = (float4((uint4(abs_code) >> uint4(16u, 20u, 24u, 28u)) & 0xFu) - 8.0f) * 0.5f; + v_lo *= 1.0f - 2.0f * float4((uint4(effective) >> uint4(0u, 1u, 2u, 3u)) & 1u); + v_hi *= 1.0f - 2.0f * float4((uint4(effective) >> uint4(4u, 5u, 6u, 7u)) & 1u); + float offset = parity != 0u ? -0.25f : 0.25f; + uint xb = x_row + (cw0 + i) * 8u; + float4 x_even = float4(float(x[xb]), float(x[xb + 2u]), float(x[xb + 4u]), float(x[xb + 6u])); + float4 x_odd = float4(float(x[xb + 1u]), float(x[xb + 3u]), float(x[xb + 5u]), float(x[xb + 7u])); + chunk_acc += dot(v_lo + offset, x_even) + dot(v_hi + offset, x_odd); + } + acc += chunk_acc * float(scales[scale_base + cw0 / codewords_per_group]); + } +} + +if (uint(LANES_PER_ROW) <= 32u) { + float total = acc; + for (uint stride = uint(LANES_PER_ROW) >> 1; stride > 0; stride >>= 1) { + total += simd_shuffle_down(total, stride); + } + if (row_lane == 0 && row_slot < uint(ROWS_PER_TG) && row < out_dim) { + out[(route * out_dim) + row] = OUT_T(total); + } +} else { + threadgroup float partials[256]; + partials[lane] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = uint(LANES_PER_ROW) >> 1; stride > 0; stride >>= 1) { + if (row_lane < stride) { + partials[lane] += partials[lane + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row_slot < uint(ROWS_PER_TG) && row < out_dim) { + out[(route * out_dim) + row] = OUT_T(partials[lane]); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/kbench.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/kbench.py new file mode 100644 index 0000000..3a0d29e --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-decode-once/kbench.py @@ -0,0 +1,38 @@ +import time, statistics, sys +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import gather_vqmm_m1_kernel_unchecked + +L = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +rng = np.random.default_rng(0) +x = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 +sc = L.scales + +def old(r, rows, tg): + return gather_vqmm_m1_kernel_unchecked(x, L.codes, sc, L.codebook, r, output_dims=2048, code_bits=16, + rows_per_threadgroup=rows, use_threadgroup_codebook=tg, use_decoded_codebook=False) +def new(r, rows, tg): + return e8p_m1.e8p_m1_token_layout(x, L.codes, sc, L.codebook, r, output_dims=2048, + rows_per_threadgroup=rows, use_threadgroup_codebook=tg) +def prod(r, rows, tg): + return L(x, r) + +def once(fn, rows, tg): + a = time.perf_counter(); mx.eval([fn(r, rows, tg) for r in routes * 4]); return (time.perf_counter() - a) / (len(routes) * 4) + +cases = [("prod", prod, 0)] + [(n, f, r) for r in (4, 8, 16, 32) for n, f in (("old", old), ("new", new))] +for n, f, r in cases: + for _ in range(3): once(f, r, True) +best = {c[0] + str(c[2]): 1e9 for c in cases} +for _ in range(20): + for n, f, r in cases: + best[n + str(r)] = min(best[n + str(r)], once(f, r, True)) +ref = np.array(prod(routes[0], 0, 0).astype(mx.float32)) +for n, f, r in cases: + ms = best[n + str(r)] * 1e3 + d = np.max(np.abs(np.array(f(routes[0], r, True).astype(mx.float32)) - ref)) / np.max(np.abs(ref)) + print("%s rows=%2d %.4f ms %.1f GB/s diff %.1e" % (n, r, ms, nbytes / (ms / 1e3) / 1e9, d)) diff --git a/benchmarks/bench_e8p_m1.py b/benchmarks/bench_e8p_m1.py index 5abe0d7..9d47f11 100644 --- a/benchmarks/bench_e8p_m1.py +++ b/benchmarks/bench_e8p_m1.py @@ -10,7 +10,7 @@ Effective bandwidth is the expert bytes one FFN reads (6 experts x 3 projections of 2-bit codes plus their scales) over the median time. Verdict for the best ``e8p_m1`` configuration: **PASS** at >= 200 GB/s, **TUNE** at -100-200 (decode the codeword once instead of per dimension, then re-measure), +100-200 (tune; decoding each codeword once was tried and is slower), **KILL** under 100 (serve on the M5 instead). Outputs of the two paths are compared on every configuration. diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index 44f303c..2f476b3 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -19,7 +19,7 @@ path is gitignored, so force-add it (`git add -f`). | # | Change | Opt-in switch | Kill test | KILL when | If it passes | |---|---|---|---|---|---| | 1 | FP8 residents kept as source bytes (weights-only mxfp8, incl. `wo_a`) | `resident_format="mxfp8"`; `--resident-format mxfp8` on the headline harness | `benchmarks/bench_dsv4_mxfp8_residents.py` | any weight differs from the bf16 decode; mxfp8 < 1.5x bf16 at M=1; > 1.1x affine-8's time | bind the composite with mxfp8 and rerun the headline series | -| 3 | Single-token E8P kernels for 16-bit codes on GPUs without NAX | `RAMP_E8P_M1=1` | `benchmarks/bench_e8p_m1.py` | outputs disagree, or < 100 GB/s effective on one routed FFN | >= 200 GB/s: make it the M4 default. 100-200: decode each codeword once, then re-measure | +| 3 | Single-token E8P kernels for 16-bit codes on GPUs without NAX | `RAMP_E8P_M1=1` | `benchmarks/bench_e8p_m1.py` | outputs disagree, or < 100 GB/s effective on one routed FFN | >= 200 GB/s: make it the M4 default. 100-200: tune (decoding each codeword once was tried and is slower) | | 5 | Headline rows also gate memory coming back in (schema v2) | always on for new rows; v1 rows keep their rule | the next headline series | — | quote only v2 rows | | 6 | One-dispatch hyper-connection sinkhorn | `RAMP_HC_SINKHORN=metal` | `benchmarks/bench_hc_sinkhorn.py` | < 3x faster at decode shape, or > 2e-6 relative difference | set it for the headline | | R3 | REAP saliency from the real gate, in the calibration pass | always collected in `calibration` mode | `produce_dsv4_teacher_cache.py reap-select --keep 128` on V4-Flash stats | median pruned-set Jaccard vs route frequency >= 0.9 (REAP adds nothing) | use REAP order for the prune and the hot set | @@ -66,5 +66,10 @@ script above runs on the Mac. - `benchmarks/bench_dsv4_mtp_headline.py` validates speculative rows against `EXPECTED_VERIFY_IMPLEMENTATION = "nax_e8p_m32n64"`, so MTP headline rows cannot validate on an M4 until the verify window has a non-NAX E8P kernel. -- The E8P M=1 branch decodes each codeword once per dimension. Decoding once - per codeword is the first optimisation if the kill test lands in TUNE. +- Decoding each E8P codeword once was tried on the M4 Max on 2026-11-05 and + made the kernel slower (113 vs 154 GB/s, kernel alone, 256 calls per sync). + The compiler already lifts the per-codeword work out of the 8-dimension loop. +- `bench_e8p_m1.py` syncs after every FFN. Real decode syncs once per token, + and with 43 dependent FFNs per sync the same kernel reaches 118 GB/s (TUNE), + against 50 GB/s for production. Its KILL verdict at 52.8 GB/s mostly + measures sync overhead. diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index e433c8a..a161550 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -91,5 +91,10 @@ or Mac-local files), plus 22 newly passing tests. - MTP headline rows require the M5-only `nax_e8p_m32n64`, so they won't validate on an M4. -- The E8P M=1 branch decodes each codeword once per dimension. Decoding once - per codeword is the next optimisation if its verdict is TUNE. +- Decoding each E8P codeword once was tried on the M4 Max on 2026-11-05 and + made the kernel slower (113 vs 154 GB/s, kernel alone, 256 calls per sync). + The compiler already lifts the per-codeword work out of the 8-dimension loop. +- `bench_e8p_m1.py` syncs after every FFN. Real decode syncs once per token, + and with 43 dependent FFNs per sync the same kernel reaches 118 GB/s (TUNE), + against 50 GB/s for production. Its KILL verdict at 52.8 GB/s mostly + measures sync overhead. diff --git a/src/ramp/kernels/e8p_m1.py b/src/ramp/kernels/e8p_m1.py index 1f943a7..cf4fb97 100644 --- a/src/ramp/kernels/e8p_m1.py +++ b/src/ramp/kernels/e8p_m1.py @@ -17,9 +17,9 @@ a separate kernel from a text substitution, so the production 8-bit kernel stays byte-identical. -The E8P branch decodes each codeword once per dimension. If the kill test -lands under its bandwidth bar, decoding a codeword once into eight values is -the first optimisation to try. +The E8P branch decodes each codeword once per dimension, but the compiler lifts +the per-codeword work out of that loop: a decode-once rewrite measured slower +on the M4 Max (113 vs 154 GB/s, 2026-11-05). """ from __future__ import annotations From 3a66e5bea1199362c0ac630e13f79e44f2ffc803 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:44:52 -0800 Subject: [PATCH 11/74] bench: time the E8P kill test as one decode step (row schema v2) v1 synced after every FFN, so its KILL at 52.8 GB/s mostly measured CPU-GPU round trips. v2 chains --ffns-per-sync dependent FFNs (default 43, one per V4-Flash layer) before one mx.eval and reports the median time per FFN. On the M4 Max, two runs both give TUNE: 122-126 GB/s best with 8-16 rows per threadgroup, 2.4-2.5x production's 50.5 GB/s, outputs within 6.4e-4. This corrects the 118 GB/s in the previous commit, whose chain script fed a [1, 6, 4096] activation forward and so sent gate and up through the per-route kernel. --- .../e8p-m1-synthetic-v2-run1.json | 63 ++++++++++++++++++ .../e8p-m1-synthetic-v2-run2.json | 63 ++++++++++++++++++ benchmarks/bench_e8p_m1.py | 66 ++++++++++++++----- .../2026-11-05-phase0-kill-tests.md | 11 ++-- .../2026-11-05-v41-plan-and-phase0-status.md | 9 +-- 5 files changed, 187 insertions(+), 25 deletions(-) create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run1.json create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run2.json diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run1.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run1.json new file mode 100644 index 0000000..b48c857 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run1.json @@ -0,0 +1,63 @@ +{ + "best": { + "effective_gb_per_s": 126.24687513292753, + "max_relative_diff_vs_production": 0.000638977624475956, + "median_ms_per_ffn": 0.30367927887032975, + "rows_per_threadgroup": 16, + "speedup_over_production": 2.4872253552301857, + "threadgroup_codebook": true + }, + "configs": [ + { + "effective_gb_per_s": 125.61180414014768, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.3052146274184938, + "rows_per_threadgroup": 8, + "speedup_over_production": 2.4747136421762974, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 126.24687513292753, + "max_relative_diff_vs_production": 0.000638977624475956, + "median_ms_per_ffn": 0.30367927887032975, + "rows_per_threadgroup": 16, + "speedup_over_production": 2.4872253552301857, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 124.18555893060979, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.3087199536736968, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.446614782349445, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 114.7250308046974, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.334177813952962, + "rows_per_threadgroup": 64, + "speedup_over_production": 2.260230244879809, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 120.24603723202925, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.31883429078017034, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.369001152341795, + "threadgroup_codebook": false + } + ], + "device": "Device(gpu, 0)", + "ffn_bytes": 38338560, + "ffns_per_sync": 43, + "mlx_version": "0.31.2", + "outputs_agree": true, + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "production_effective_gb_per_s": 50.758116817783794, + "production_median_ms_per_ffn": 0.7553188022643025, + "record_type": "ramp_e8p_m1_kill_test_v2", + "source": "synthetic", + "verdict": "TUNE" +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run2.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run2.json new file mode 100644 index 0000000..33bdef8 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-run2.json @@ -0,0 +1,63 @@ +{ + "best": { + "effective_gb_per_s": 123.71473313984096, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.30989486075731987, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.4497149387742128, + "threadgroup_codebook": true + }, + "configs": [ + { + "effective_gb_per_s": 122.3088833198184, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.3134568721370035, + "rows_per_threadgroup": 8, + "speedup_over_production": 2.421877257534667, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 121.26227485943866, + "max_relative_diff_vs_production": 0.000638977624475956, + "median_ms_per_ffn": 0.3161623022860176, + "rows_per_threadgroup": 16, + "speedup_over_production": 2.4011530291798957, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 123.71473313984096, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.30989486075731987, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.4497149387742128, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 114.24601252337658, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.33557897692188876, + "rows_per_threadgroup": 64, + "speedup_over_production": 2.2622217780444154, + "threadgroup_codebook": true + }, + { + "effective_gb_per_s": 120.92889489763162, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.3170339068462856, + "rows_per_threadgroup": 32, + "speedup_over_production": 2.394551666092416, + "threadgroup_codebook": false + } + ], + "device": "Device(gpu, 0)", + "ffn_bytes": 38338560, + "ffns_per_sync": 43, + "mlx_version": "0.31.2", + "outputs_agree": true, + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "production_effective_gb_per_s": 50.50168539272789, + "production_median_ms_per_ffn": 0.759154069846561, + "record_type": "ramp_e8p_m1_kill_test_v2", + "source": "synthetic", + "verdict": "TUNE" +} diff --git a/benchmarks/bench_e8p_m1.py b/benchmarks/bench_e8p_m1.py index 9d47f11..278a030 100644 --- a/benchmarks/bench_e8p_m1.py +++ b/benchmarks/bench_e8p_m1.py @@ -8,7 +8,10 @@ and threadgroup codebook staging. Effective bandwidth is the expert bytes one FFN reads (6 experts x 3 -projections of 2-bit codes plus their scales) over the median time. Verdict +projections of 2-bit codes plus their scales) over the median time per FFN. +Each timed sample chains ``--ffns-per-sync`` dependent FFNs (default 43, one +per V4-Flash layer) before one ``mx.eval``, as one decode step does; syncing +after every FFN measured mostly CPU-GPU round trips (v1 rows). Verdict for the best ``e8p_m1`` configuration: **PASS** at >= 200 GB/s, **TUNE** at 100-200 (tune; decoding each codeword once was tried and is slower), **KILL** under 100 (serve on the M5 instead). Outputs of the two paths are @@ -72,18 +75,39 @@ def _ffn(layers, x, routes, intermediate): return layers["down_proj"](act.reshape(1, routes.shape[-1], intermediate), routes) -def _median_ms(fn, routes_list, iterations, warmup) -> float: - for i in range(warmup): - mx.eval(fn(routes_list[i % len(routes_list)])) - samples = [] - for i in range(iterations): - started = time.perf_counter() - mx.eval(fn(routes_list[i % len(routes_list)])) - samples.append((time.perf_counter() - started) * 1e3) - return statistics.median(samples) +def _chain(layers, x, routes_seq, intermediate): + """Dependent FFNs, each feeding the next like consecutive decoder layers.""" + + for routes in routes_seq: + y = _ffn(layers, x, routes, intermediate) # [1, k, hidden] + x = (x + y.sum(axis=1) * 1e-3).astype(x.dtype) + return x + +def _median_ms(layers, x, routes_list, intermediate, iterations, warmup, ffns_per_sync) -> float: + """Median ms per FFN, with ``ffns_per_sync`` chained FFNs per ``mx.eval``.""" -def run(layers, *, top_k: int, iterations: int, warmup: int, seed: int, sweep: Sequence[tuple[int, bool]]) -> dict: + def sample(i): + seq = [routes_list[(i * ffns_per_sync + j) % len(routes_list)] for j in range(ffns_per_sync)] + started = time.perf_counter() + mx.eval(_chain(layers, x, seq, intermediate)) + return (time.perf_counter() - started) * 1e3 / ffns_per_sync + + for i in range(warmup): + sample(i) + return statistics.median(sample(i) for i in range(iterations)) + + +def run( + layers, + *, + top_k: int, + iterations: int, + warmup: int, + seed: int, + sweep: Sequence[tuple[int, bool]], + ffns_per_sync: int = 43, +) -> dict: rng = np.random.default_rng(seed) experts = layers["gate_proj"].num_experts hidden = layers["gate_proj"].input_dims @@ -97,7 +121,7 @@ def run(layers, *, top_k: int, iterations: int, warmup: int, seed: int, sweep: S ffn_bytes = top_k * sum(projection_bytes_per_expert(layers[p]) for p in PROJECTIONS) os.environ[e8p_m1.E8P_M1_ENV] = "0" - production_ms = _median_ms(lambda r: _ffn(layers, x, r, intermediate), routes_list, iterations, warmup) + production_ms = _median_ms(layers, x, routes_list, intermediate, iterations, warmup, ffns_per_sync) reference = [np.array(_ffn(layers, x, r, intermediate).astype(mx.float32)) for r in routes_list[:4]] configs = [] @@ -122,7 +146,7 @@ def run(layers, *, top_k: int, iterations: int, warmup: int, seed: int, sweep: S float(np.max(np.abs(g - w)) / max(float(np.max(np.abs(w))), 1e-12)) for g, w in zip(got, reference) ) - ms = _median_ms(lambda r: _ffn(layers, x, r, intermediate), routes_list, iterations, warmup) + ms = _median_ms(layers, x, routes_list, intermediate, iterations, warmup, ffns_per_sync) configs.append( { "rows_per_threadgroup": rows_per_tg, @@ -144,6 +168,7 @@ def run(layers, *, top_k: int, iterations: int, warmup: int, seed: int, sweep: S verdict = "KILL" if not agrees or gb_s < KILL_GB_S else ("PASS" if gb_s >= PASS_GB_S else "TUNE") return { "ffn_bytes": ffn_bytes, + "ffns_per_sync": ffns_per_sync, "production_median_ms_per_ffn": production_ms, "production_effective_gb_per_s": ffn_bytes / (production_ms / 1e3) / 1e9, "configs": configs, @@ -162,7 +187,8 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--intermediate", type=int, default=2048) parser.add_argument("--group-size", type=int, default=512) parser.add_argument("--top-k", type=int, default=6) - parser.add_argument("--iterations", type=int, default=200) + parser.add_argument("--iterations", type=int, default=50, help="timed samples, each --ffns-per-sync FFNs") + parser.add_argument("--ffns-per-sync", type=int, default=43, help="chained FFNs per mx.eval (V4-Flash has 43 layers)") parser.add_argument("--warmup", type=int, default=20) parser.add_argument("--seed", type=int, default=20261105) parser.add_argument("--sweep", default="8:1,16:1,32:1,64:1,32:0", help="rows_per_threadgroup:tg_codebook,...") @@ -192,12 +218,20 @@ def main(argv: Sequence[str] | None = None) -> int: rows, codebook = item.split(":") sweep.append((int(rows), codebook.strip() == "1")) result = { - "record_type": "ramp_e8p_m1_kill_test_v1", + "record_type": "ramp_e8p_m1_kill_test_v2", "device": str(mx.default_device()), "mlx_version": mx.__version__, "platform": platform.platform(), "source": "synthetic" if args.artifact_dir is None else str(args.artifact_dir), - **run(layers, top_k=args.top_k, iterations=args.iterations, warmup=args.warmup, seed=args.seed, sweep=sweep), + **run( + layers, + top_k=args.top_k, + iterations=args.iterations, + warmup=args.warmup, + seed=args.seed, + sweep=sweep, + ffns_per_sync=args.ffns_per_sync, + ), } text = json.dumps(result, indent=2, sort_keys=True) if args.output_json is not None: diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index 2f476b3..fa039ef 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -19,7 +19,7 @@ path is gitignored, so force-add it (`git add -f`). | # | Change | Opt-in switch | Kill test | KILL when | If it passes | |---|---|---|---|---|---| | 1 | FP8 residents kept as source bytes (weights-only mxfp8, incl. `wo_a`) | `resident_format="mxfp8"`; `--resident-format mxfp8` on the headline harness | `benchmarks/bench_dsv4_mxfp8_residents.py` | any weight differs from the bf16 decode; mxfp8 < 1.5x bf16 at M=1; > 1.1x affine-8's time | bind the composite with mxfp8 and rerun the headline series | -| 3 | Single-token E8P kernels for 16-bit codes on GPUs without NAX | `RAMP_E8P_M1=1` | `benchmarks/bench_e8p_m1.py` | outputs disagree, or < 100 GB/s effective on one routed FFN | >= 200 GB/s: make it the M4 default. 100-200: tune (decoding each codeword once was tried and is slower) | +| 3 | Single-token E8P kernels for 16-bit codes on GPUs without NAX | `RAMP_E8P_M1=1` | `benchmarks/bench_e8p_m1.py` | outputs disagree, or < 100 GB/s effective per routed FFN, 43 chained FFNs per sync | >= 200 GB/s: make it the M4 default. 100-200: tune (decoding each codeword once was tried and is slower) | | 5 | Headline rows also gate memory coming back in (schema v2) | always on for new rows; v1 rows keep their rule | the next headline series | — | quote only v2 rows | | 6 | One-dispatch hyper-connection sinkhorn | `RAMP_HC_SINKHORN=metal` | `benchmarks/bench_hc_sinkhorn.py` | < 3x faster at decode shape, or > 2e-6 relative difference | set it for the headline | | R3 | REAP saliency from the real gate, in the calibration pass | always collected in `calibration` mode | `produce_dsv4_teacher_cache.py reap-select --keep 128` on V4-Flash stats | median pruned-set Jaccard vs route frequency >= 0.9 (REAP adds nothing) | use REAP order for the prune and the hot set | @@ -69,7 +69,8 @@ script above runs on the Mac. - Decoding each E8P codeword once was tried on the M4 Max on 2026-11-05 and made the kernel slower (113 vs 154 GB/s, kernel alone, 256 calls per sync). The compiler already lifts the per-codeword work out of the 8-dimension loop. -- `bench_e8p_m1.py` syncs after every FFN. Real decode syncs once per token, - and with 43 dependent FFNs per sync the same kernel reaches 118 GB/s (TUNE), - against 50 GB/s for production. Its KILL verdict at 52.8 GB/s mostly - measures sync overhead. +- `bench_e8p_m1.py` v1 synced after every FFN and called KILL at 52.8 GB/s, + mostly CPU-GPU round trips. v2 (`ramp_e8p_m1_kill_test_v2`) chains 43 + dependent FFNs per `mx.eval`, as one decode step does. On the M4 Max it + gives **TUNE**: 122-126 GB/s best (8-16 rows per threadgroup), 2.4-2.5x + production's 50.5 GB/s, outputs within 6.4e-4. Quote only v2 rows. diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index a161550..80b2b87 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -94,7 +94,8 @@ or Mac-local files), plus 22 newly passing tests. - Decoding each E8P codeword once was tried on the M4 Max on 2026-11-05 and made the kernel slower (113 vs 154 GB/s, kernel alone, 256 calls per sync). The compiler already lifts the per-codeword work out of the 8-dimension loop. -- `bench_e8p_m1.py` syncs after every FFN. Real decode syncs once per token, - and with 43 dependent FFNs per sync the same kernel reaches 118 GB/s (TUNE), - against 50 GB/s for production. Its KILL verdict at 52.8 GB/s mostly - measures sync overhead. +- `bench_e8p_m1.py` v1 synced after every FFN and called KILL at 52.8 GB/s, + mostly CPU-GPU round trips. v2 (`ramp_e8p_m1_kill_test_v2`) chains 43 + dependent FFNs per `mx.eval`, as one decode step does. On the M4 Max it + gives **TUNE**: 122-126 GB/s best (8-16 rows per threadgroup), 2.4-2.5x + production's 50.5 GB/s, outputs within 6.4e-4. Quote only v2 rows. From bff76f2a9866b814754548f83b310575da6f68f2 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:50:17 -0800 Subject: [PATCH 12/74] bench: time the mxfp8 resident kill test as one decode step (v2) v1 synced after every sub-millisecond matmul, so it mostly measured CPU-GPU round trips and called KILL (mxfp8 0.88x bf16). v2 chains --calls-per-sync dependent calls (default 43, one per V4-Flash layer) before one mx.eval; each call's activation depends on the previous output so the GPU cannot overlap them. On the M4 Max both runs PASS: mxfp8 is 1.66x bf16 and takes 1.01-1.05x affine-8's time, still bit-exact at all eight shapes. The large tensors read at 350-450 GB/s. --- .../mxfp8-residents-synthetic-v2-run1.json | 429 ++++++++++++++++++ .../mxfp8-residents-synthetic-v2-run2.json | 429 ++++++++++++++++++ benchmarks/bench_dsv4_mxfp8_residents.py | 60 ++- .../2026-11-05-phase0-kill-tests.md | 5 + .../2026-11-05-v41-plan-and-phase0-status.md | 5 + 5 files changed, 914 insertions(+), 14 deletions(-) create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run1.json create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run2.json diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run1.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run1.json new file mode 100644 index 0000000..507b596 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run1.json @@ -0,0 +1,429 @@ +{ + "created_utc": "2026-11-06T03:49:50+00:00", + "device": "Device(gpu, 0)", + "device_info": { + "architecture": "applegpu_g16s", + "device_name": "Apple M4 Max", + "max_buffer_length": 86586540032, + "max_recommended_working_set_size": 115448725504, + "memory_size": 137438953472, + "resource_limit": 499000 + }, + "exactness": { + "attn.wkv": { + "bf16_bytes": 4194304, + "bit_mismatches": 0, + "elements": 2097152, + "mxfp8_bytes": 2162688 + }, + "attn.wo_a": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "attn.wo_b": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "attn.wq_a": { + "bf16_bytes": 8388608, + "bit_mismatches": 0, + "elements": 4194304, + "mxfp8_bytes": 4325376 + }, + "attn.wq_b": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "ffn.shared_experts.w1": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + }, + "ffn.shared_experts.w2": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + }, + "ffn.shared_experts.w3": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + } + }, + "layer": null, + "mlx_version": "0.31.2", + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "record_type": "dsv4_mxfp8_resident_kill_test_v2", + "settings": { + "calls_per_sync": 43, + "copies": 8, + "iterations": 20, + "rounds": 5, + "seed": 20261105, + "warmup": 3 + }, + "shapes": { + "attn.wkv": [ + 512, + 4096 + ], + "attn.wo_a": [ + 8192, + 4096 + ], + "attn.wo_b": [ + 4096, + 8192 + ], + "attn.wq_a": [ + 1024, + 4096 + ], + "attn.wq_b": [ + 32768, + 1024 + ], + "ffn.shared_experts.w1": [ + 2048, + 4096 + ], + "ffn.shared_experts.w2": [ + 4096, + 2048 + ], + "ffn.shared_experts.w3": [ + 2048, + 4096 + ] + }, + "source": "synthetic", + "timings": { + "attn.wkv": { + "affine8_g64": { + "effective_gb_per_s": 117.88515055513322, + "median_ms_per_call": 0.018901651221609046, + "round_ms": [ + 0.019171511413269608, + 0.018474813767297323, + 0.01889244188976929, + 0.019241279047957166, + 0.018901651221609046 + ], + "weight_bytes": 2228224 + }, + "bf16": { + "effective_gb_per_s": 165.36217826043818, + "median_ms_per_call": 0.02536434899517443, + "round_ms": [ + 0.02575241845523462, + 0.02498158150850687, + 0.024488860765623663, + 0.02536434899517443, + 0.025727721147759015 + ], + "weight_bytes": 4194304 + }, + "mxfp8": { + "effective_gb_per_s": 103.28533410732432, + "median_ms_per_call": 0.02093896503982589, + "round_ms": [ + 0.021193325749685077, + 0.020817825760215868, + 0.020693313820439194, + 0.021076557797726332, + 0.02093896503982589 + ], + "weight_bytes": 2162688 + } + }, + "attn.wo_a": { + "affine8_g64": { + "effective_gb_per_s": 377.4318395226111, + "median_ms_per_call": 0.09445833728572917, + "round_ms": [ + 0.09079940710487494, + 0.10062548836833948, + 0.09445833728572917, + 0.09613808110453795, + 0.0907858487808817 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 400.91202351724274, + "median_ms_per_call": 0.1673904998190051, + "round_ms": [ + 0.17180233695652597, + 0.1666923490120098, + 0.16632412768748783, + 0.17667877918304226, + 0.1673904998190051 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 359.651200009676, + "median_ms_per_call": 0.09621268606658073, + "round_ms": [ + 0.09504505832473811, + 0.09621268606658073, + 0.10114632578036034, + 0.1009491395454334, + 0.09025629071063947 + ], + "weight_bytes": 34603008 + } + }, + "attn.wo_b": { + "affine8_g64": { + "effective_gb_per_s": 374.3957633791575, + "median_ms_per_call": 0.09522432539893617, + "round_ms": [ + 0.08996367436931144, + 0.09794525604500154, + 0.09233043035004981, + 0.09522432539893617, + 0.09756298839970123 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 384.7842163658154, + "median_ms_per_call": 0.17440648848288368, + "round_ms": [ + 0.17165697652164327, + 0.17440648848288368, + 0.17524320905659954, + 0.17554553490955122, + 0.172934593053448 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 370.70419374851963, + "median_ms_per_call": 0.09334398850495385, + "round_ms": [ + 0.08617103476142293, + 0.09418702359462894, + 0.09601744200471182, + 0.09334398850495385, + 0.08957752339314495 + ], + "weight_bytes": 34603008 + } + }, + "attn.wq_a": { + "affine8_g64": { + "effective_gb_per_s": 194.17178634635061, + "median_ms_per_call": 0.022951058358452174, + "round_ms": [ + 0.0277771282843639, + 0.02407123298156833, + 0.022951058358452174, + 0.021333337171263126, + 0.021545058062170136 + ], + "weight_bytes": 4456448 + }, + "bf16": { + "effective_gb_per_s": 272.3368393730946, + "median_ms_per_call": 0.03080232560277245, + "round_ms": [ + 0.044809104766436784, + 0.03175581379003043, + 0.03080232560277245, + 0.030590604711865442, + 0.03065068652409367 + ], + "weight_bytes": 8388608 + }, + "mxfp8": { + "effective_gb_per_s": 178.17737895723826, + "median_ms_per_call": 0.02427567419227819, + "round_ms": [ + 0.037029069823992636, + 0.030775197787037075, + 0.02427567419227819, + 0.023396325644192306, + 0.02350194209125326 + ], + "weight_bytes": 4325376 + } + }, + "attn.wq_b": { + "affine8_g64": { + "effective_gb_per_s": 430.18001625193773, + "median_ms_per_call": 0.08287596506835504, + "round_ms": [ + 0.08287596506835504, + 0.0827180233182992, + 0.08377180229006118, + 0.08250193042290765, + 0.09378973295814683 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 451.4740676890608, + "median_ms_per_call": 0.14864389519315474, + "round_ms": [ + 0.14864389519315474, + 0.15096608136488168, + 0.14745058129043426, + 0.14806202332608315, + 0.18916956959887904 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 417.38857949296505, + "median_ms_per_call": 0.08290358121929214, + "round_ms": [ + 0.08290358121929214, + 0.08399224451602286, + 0.08090261605481595, + 0.08119670924633135, + 0.08568072083763519 + ], + "weight_bytes": 34603008 + } + }, + "ffn.shared_experts.w1": { + "affine8_g64": { + "effective_gb_per_s": 270.20676991221814, + "median_ms_per_call": 0.03298546517874266, + "round_ms": [ + 0.03286337224106023, + 0.03354796540089558, + 0.03319427907077033, + 0.03205620931450624, + 0.03298546517874266 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 353.94466951119, + "median_ms_per_call": 0.0474006743007881, + "round_ms": [ + 0.04676745346270848, + 0.04999709334549342, + 0.048844488256416005, + 0.0474006743007881, + 0.047367255996674476 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 265.70166976009136, + "median_ms_per_call": 0.032558139389229204, + "round_ms": [ + 0.03290165122327676, + 0.0334568836098147, + 0.03255086038586532, + 0.03223401172151573, + 0.032558139389229204 + ], + "weight_bytes": 8650752 + } + }, + "ffn.shared_experts.w2": { + "affine8_g64": { + "effective_gb_per_s": 283.11890788381345, + "median_ms_per_call": 0.031481104764849124, + "round_ms": [ + 0.031481104764849124, + 0.031687499993199184, + 0.031428779461854246, + 0.03148061642964739, + 0.03174175562466993 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 351.02757450305774, + "median_ms_per_call": 0.047794581447771295, + "round_ms": [ + 0.04740746530736688, + 0.04811966311197381, + 0.047794581447771295, + 0.04782510459758774, + 0.04771656998389855 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 279.2398129187639, + "median_ms_per_call": 0.03097965118074573, + "round_ms": [ + 0.030670534998048532, + 0.03064632534894139, + 0.03097965118074573, + 0.03115019759672239, + 0.03109349995082634 + ], + "weight_bytes": 8650752 + } + }, + "ffn.shared_experts.w3": { + "affine8_g64": { + "effective_gb_per_s": 275.7845823914847, + "median_ms_per_call": 0.032318325856765515, + "round_ms": [ + 0.03201065099050919, + 0.031834302421356014, + 0.03246802343921952, + 0.032318325856765515, + 0.03248691825510197 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 354.386644461155, + "median_ms_per_call": 0.04734155832963108, + "round_ms": [ + 0.047691360313640255, + 0.04733527867058509, + 0.04734155832963108, + 0.04790067438545182, + 0.0469578605356388 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 267.56901299741423, + "median_ms_per_call": 0.032330918678104184, + "round_ms": [ + 0.0321501974276332, + 0.0324093954648476, + 0.03353826739543746, + 0.032330918678104184, + 0.03195251168975563 + ], + "weight_bytes": 8650752 + } + } + }, + "verdict": { + "bit_mismatches": 0, + "exact": true, + "mxfp8_speedup_over_bf16": 1.6664370215227893, + "mxfp8_time_over_affine8": 1.005708639691768, + "reasons": [], + "speed_valid": true, + "summed_median_ms": { + "affine8_g64": 0.4111962331334389, + "bf16": 0.6891443721711809, + "mxfp8": 0.41354360427100995 + }, + "verdict": "PASS" + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run2.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run2.json new file mode 100644 index 0000000..29242fc --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/mxfp8-residents-synthetic-v2-run2.json @@ -0,0 +1,429 @@ +{ + "created_utc": "2026-11-06T03:50:00+00:00", + "device": "Device(gpu, 0)", + "device_info": { + "architecture": "applegpu_g16s", + "device_name": "Apple M4 Max", + "max_buffer_length": 86586540032, + "max_recommended_working_set_size": 115448725504, + "memory_size": 137438953472, + "resource_limit": 499000 + }, + "exactness": { + "attn.wkv": { + "bf16_bytes": 4194304, + "bit_mismatches": 0, + "elements": 2097152, + "mxfp8_bytes": 2162688 + }, + "attn.wo_a": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "attn.wo_b": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "attn.wq_a": { + "bf16_bytes": 8388608, + "bit_mismatches": 0, + "elements": 4194304, + "mxfp8_bytes": 4325376 + }, + "attn.wq_b": { + "bf16_bytes": 67108864, + "bit_mismatches": 0, + "elements": 33554432, + "mxfp8_bytes": 34603008 + }, + "ffn.shared_experts.w1": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + }, + "ffn.shared_experts.w2": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + }, + "ffn.shared_experts.w3": { + "bf16_bytes": 16777216, + "bit_mismatches": 0, + "elements": 8388608, + "mxfp8_bytes": 8650752 + } + }, + "layer": null, + "mlx_version": "0.31.2", + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "record_type": "dsv4_mxfp8_resident_kill_test_v2", + "settings": { + "calls_per_sync": 43, + "copies": 8, + "iterations": 20, + "rounds": 5, + "seed": 20261105, + "warmup": 3 + }, + "shapes": { + "attn.wkv": [ + 512, + 4096 + ], + "attn.wo_a": [ + 8192, + 4096 + ], + "attn.wo_b": [ + 4096, + 8192 + ], + "attn.wq_a": [ + 1024, + 4096 + ], + "attn.wq_b": [ + 32768, + 1024 + ], + "ffn.shared_experts.w1": [ + 2048, + 4096 + ], + "ffn.shared_experts.w2": [ + 4096, + 2048 + ], + "ffn.shared_experts.w3": [ + 2048, + 4096 + ] + }, + "source": "synthetic", + "timings": { + "attn.wkv": { + "affine8_g64": { + "effective_gb_per_s": 123.19335756867, + "median_ms_per_call": 0.018087208953274543, + "round_ms": [ + 0.017982069335249797, + 0.01816520924939839, + 0.018087208953274543, + 0.018070732124236433, + 0.018106604625774157 + ], + "weight_bytes": 2228224 + }, + "bf16": { + "effective_gb_per_s": 165.3812097704874, + "median_ms_per_call": 0.02536143015171293, + "round_ms": [ + 0.025331395337098214, + 0.025791662940105726, + 0.025838662580543653, + 0.025216569896614135, + 0.02536143015171293 + ], + "weight_bytes": 4194304 + }, + "mxfp8": { + "effective_gb_per_s": 101.77589158529817, + "median_ms_per_call": 0.021249511709631703, + "round_ms": [ + 0.021006302166253675, + 0.020820244085476843, + 0.021312011508704272, + 0.021485941752596476, + 0.021249511709631703 + ], + "weight_bytes": 2162688 + } + }, + "attn.wo_a": { + "affine8_g64": { + "effective_gb_per_s": 389.6368286215456, + "median_ms_per_call": 0.09149952309725937, + "round_ms": [ + 0.09149952309725937, + 0.09543944230075752, + 0.08811966311673872, + 0.10774080197565084, + 0.08725677887627552 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 404.80547814406657, + "median_ms_per_call": 0.16578052329646725, + "round_ms": [ + 0.16559400023123638, + 0.1665716977291849, + 0.16527616271348455, + 0.18161095355280005, + 0.16578052329646725 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 365.816813102497, + "median_ms_per_call": 0.0945910815485255, + "round_ms": [ + 0.0945910815485255, + 0.09285950042350694, + 0.09141472062680783, + 0.1081787790902782, + 0.09522288374407867 + ], + "weight_bytes": 34603008 + } + }, + "attn.wo_b": { + "affine8_g64": { + "effective_gb_per_s": 387.107456589037, + "median_ms_per_call": 0.09209738379658394, + "round_ms": [ + 0.09140310447587263, + 0.09209738379658394, + 0.08843895345709699, + 0.12042149996146734, + 0.12089680206182218 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 384.48089172054546, + "median_ms_per_call": 0.1745440812407841, + "round_ms": [ + 0.1764244188115997, + 0.16961869753918848, + 0.16992925605357645, + 0.1961807209354024, + 0.1745440812407841 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 348.2641136251952, + "median_ms_per_call": 0.09935852316164867, + "round_ms": [ + 0.09935852316164867, + 0.09093652303373917, + 0.09315649993706841, + 0.12642346514757116, + 0.13854118642061514 + ], + "weight_bytes": 34603008 + } + }, + "attn.wq_a": { + "affine8_g64": { + "effective_gb_per_s": 194.24146461236384, + "median_ms_per_call": 0.02294282535859925, + "round_ms": [ + 0.023552802388642935, + 0.023326546841755856, + 0.02294282535859925, + 0.021597860194201214, + 0.021567348888966928 + ], + "weight_bytes": 4456448 + }, + "bf16": { + "effective_gb_per_s": 269.53868627697466, + "median_ms_per_call": 0.031122092772166918, + "round_ms": [ + 0.03372044214534811, + 0.031814918931855195, + 0.031122092772166918, + 0.03086579090449959, + 0.03101744216617717 + ], + "weight_bytes": 8388608 + }, + "mxfp8": { + "effective_gb_per_s": 177.73748468501907, + "median_ms_per_call": 0.024335755666089788, + "round_ms": [ + 0.028541662898131235, + 0.026405999791635156, + 0.024335755666089788, + 0.024085278963930038, + 0.02344090695936926 + ], + "weight_bytes": 4325376 + } + }, + "attn.wq_b": { + "affine8_g64": { + "effective_gb_per_s": 422.57369042139044, + "median_ms_per_call": 0.08436773232249326, + "round_ms": [ + 0.0854161856316983, + 0.08461288336209606, + 0.08425774420936441, + 0.08436773232249326, + 0.08346753494088474 + ], + "weight_bytes": 35651584 + }, + "bf16": { + "effective_gb_per_s": 410.1321441161742, + "median_ms_per_call": 0.16362741853511176, + "round_ms": [ + 0.16440843031633384, + 0.16242732547039468, + 0.16278100029435444, + 0.16418846525782504, + 0.16362741853511176 + ], + "weight_bytes": 67108864 + }, + "mxfp8": { + "effective_gb_per_s": 387.49845290049126, + "median_ms_per_call": 0.08929844168664584, + "round_ms": [ + 0.0915271395866131, + 0.08929844168664584, + 0.08705813948032555, + 0.09061433753908374, + 0.08242344179158206 + ], + "weight_bytes": 34603008 + } + }, + "ffn.shared_experts.w1": { + "affine8_g64": { + "effective_gb_per_s": 284.94760510813404, + "median_ms_per_call": 0.03127906969640144, + "round_ms": [ + 0.03127906969640144, + 0.030744174457436733, + 0.031008244002086308, + 0.03279989543316756, + 0.03204747681170269 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 354.70603402628257, + "median_ms_per_call": 0.04729893035526106, + "round_ms": [ + 0.04660512752516932, + 0.04780716310136107, + 0.04729893035526106, + 0.04804845348401212, + 0.046610465032308424 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 273.0814011550195, + "median_ms_per_call": 0.03167829066135942, + "round_ms": [ + 0.031283430194720456, + 0.03167829066135942, + 0.03128827936665783, + 0.031721418477096704, + 0.03222287206120009 + ], + "weight_bytes": 8650752 + } + }, + "ffn.shared_experts.w2": { + "affine8_g64": { + "effective_gb_per_s": 281.3307754453691, + "median_ms_per_call": 0.031681197998655396, + "round_ms": [ + 0.031019860829854775, + 0.03179651177434121, + 0.03254069773436979, + 0.030814441595074918, + 0.031681197998655396 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 351.82300887874, + "median_ms_per_call": 0.04768652298628504, + "round_ms": [ + 0.04734787216875702, + 0.047846418753981074, + 0.047673453335910165, + 0.04843604645345273, + 0.04768652298628504 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 279.4058272822925, + "median_ms_per_call": 0.03096124402323175, + "round_ms": [ + 0.030500488761855765, + 0.030575569876101473, + 0.03096124402323175, + 0.031158907425611518, + 0.03127859320578187 + ], + "weight_bytes": 8650752 + } + }, + "ffn.shared_experts.w3": { + "affine8_g64": { + "effective_gb_per_s": 279.9692533063395, + "median_ms_per_call": 0.031835267247177314, + "round_ms": [ + 0.031857558073974106, + 0.03157461638735651, + 0.03156395355607708, + 0.031835267247177314, + 0.032773244108051755 + ], + "weight_bytes": 8912896 + }, + "bf16": { + "effective_gb_per_s": 357.8732852182121, + "median_ms_per_call": 0.046880325223969, + "round_ms": [ + 0.04709204645329264, + 0.0463788838228644, + 0.04638033698388744, + 0.046880325223969, + 0.04720494190371747 + ], + "weight_bytes": 16777216 + }, + "mxfp8": { + "effective_gb_per_s": 269.8172499246007, + "median_ms_per_call": 0.03206152313248103, + "round_ms": [ + 0.03181781408615237, + 0.03169282565575613, + 0.032378871976614515, + 0.03206152313248103, + 0.032399220968769904 + ], + "weight_bytes": 8650752 + } + } + }, + "verdict": { + "bit_mismatches": 0, + "exact": true, + "mxfp8_speedup_over_bf16": 1.6581920421850846, + "mxfp8_time_over_affine8": 1.0488970824576456, + "reasons": [], + "speed_valid": true, + "summed_median_ms": { + "affine8_g64": 0.4037902084704445, + "bf16": 0.7023013245617581, + "mxfp8": 0.4235343715896137 + }, + "verdict": "PASS" + } +} diff --git a/benchmarks/bench_dsv4_mxfp8_residents.py b/benchmarks/bench_dsv4_mxfp8_residents.py index cbfcae5..1c547c1 100644 --- a/benchmarks/bench_dsv4_mxfp8_residents.py +++ b/benchmarks/bench_dsv4_mxfp8_residents.py @@ -13,7 +13,10 @@ Verdict, from the ranked plan: **KILL** if any bit differs, if mxfp8 is less than 1.5x the bf16 speed, or if it is more than 10% slower than affine-8. Speed is judged on the summed per-call medians across the chosen tensors, the -way one decode step reads them. +way one decode step reads them. Each timed sample chains ``--calls-per-sync`` +dependent calls (default 43, one per V4-Flash layer) before one ``mx.eval``, +as one decode step does; v1 synced after every sub-millisecond call and so +mostly measured CPU-GPU round trips. Inputs: a resident package or the source checkpoint (``--checkpoint``, a directory with ``model.safetensors.index.json``), or ``--synthetic`` FP8 @@ -50,7 +53,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] HEAVY_JOB_LOCK = REPO_ROOT / ".keep-heavy-job.lock" -RECORD_TYPE = "dsv4_mxfp8_resident_kill_test_v1" +RECORD_TYPE = "dsv4_mxfp8_resident_kill_test_v2" FP8_BLOCK = (128, 128) #: DeepSeek-V4-Flash-0731 resident shapes (out, in) as they ship, from the @@ -151,14 +154,31 @@ def check_exactness(codes: np.ndarray, scales: np.ndarray) -> dict[str, Any]: def _median_call_ms( - fn: Callable[[int], mx.array], *, iterations: int, warmup: int + fn: Callable[[int, mx.array], mx.array], + x: mx.array, + *, + iterations: int, + warmup: int, + calls_per_sync: int, ) -> float: + """Median ms per call over ``iterations`` samples of chained calls. + + Each call's activation depends on the previous call's output, like + consecutive layers in one decode step, so the GPU cannot overlap them. + """ + + def sample(index: int) -> float: + started = time.perf_counter() + y = x + for j in range(calls_per_sync): + out = fn(index * calls_per_sync + j, y) + y = y + out.reshape(-1)[:1].astype(y.dtype) * 1e-6 + mx.eval(y) + return (time.perf_counter() - started) * 1_000.0 / calls_per_sync + for i in range(warmup): - mx.eval(fn(i)) - started = time.perf_counter() - for i in range(iterations): - mx.eval(fn(i)) - return (time.perf_counter() - started) * 1_000.0 / iterations + sample(i) + return statistics.median(sample(i) for i in range(iterations)) def time_variants( @@ -171,6 +191,7 @@ def time_variants( rounds: int, seed: int, groups: int = 1, + calls_per_sync: int = 43, ) -> dict[str, Any]: """Per-call M=1 latency for bf16, mxfp8 and affine-8 g64, interleaved. @@ -201,16 +222,16 @@ def time_variants( x_shape = (1, groups, 1, cols) if groups > 1 else (1, cols) x = mx.random.normal(x_shape, key=mx.random.key(seed)).astype(mx.bfloat16) - def bf16(i: int) -> mx.array: + def bf16(i: int, x: mx.array) -> mx.array: return x @ bf16_copies[i % copies].swapaxes(-1, -2) - def mxfp8(i: int) -> mx.array: + def mxfp8(i: int, x: mx.array) -> mx.array: j = i % copies return mx.quantized_matmul( x, packed_copies[j], scale_copies[j], transpose=True, group_size=32, bits=8, mode="mxfp8" ) - def affine8(i: int) -> mx.array: + def affine8(i: int, x: mx.array) -> mx.array: w, s, b = affine_copies[i % copies] return mx.quantized_matmul(x, w, s, b, transpose=True, group_size=64, bits=8) @@ -222,7 +243,13 @@ def affine8(i: int) -> mx.array: rotated = order[round_index % len(order) :] + order[: round_index % len(order)] for name in rotated: samples[name].append( - _median_call_ms(variants[name], iterations=iterations, warmup=warmup) + _median_call_ms( + variants[name], + x, + iterations=iterations, + warmup=warmup, + calls_per_sync=calls_per_sync, + ) ) weight_bytes = { "bf16": rows * cols * 2, @@ -304,6 +331,7 @@ def run( rounds: int, seed: int, exactness_only: bool, + calls_per_sync: int = 43, ) -> dict[str, Any]: if (checkpoint is None) == (synthetic_shapes is None): raise ValueError("pass exactly one of checkpoint or synthetic_shapes") @@ -330,6 +358,7 @@ def run( rounds=rounds, seed=seed, groups=GROUPED_TENSORS.get(tensor, 1), + calls_per_sync=calls_per_sync, ) mx.clear_cache() return { @@ -350,6 +379,7 @@ def run( "warmup": warmup, "rounds": rounds, "seed": seed, + "calls_per_sync": calls_per_sync, }, "exactness": exactness, "timings": timings, @@ -365,8 +395,9 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--layer", type=int, default=20) parser.add_argument("--tensors", default=",".join(V4_FLASH_RESIDENT_SHAPES)) parser.add_argument("--copies", type=int, default=8) - parser.add_argument("--iterations", type=int, default=200) - parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iterations", type=int, default=20, help="timed samples per round") + parser.add_argument("--calls-per-sync", type=int, default=43, help="chained calls per mx.eval") + parser.add_argument("--warmup", type=int, default=3) parser.add_argument("--rounds", type=int, default=5) parser.add_argument("--seed", type=int, default=20261105) parser.add_argument("--exactness-only", action="store_true") @@ -394,6 +425,7 @@ def _go() -> dict[str, Any]: rounds=args.rounds, seed=args.seed, exactness_only=args.exactness_only, + calls_per_sync=args.calls_per_sync, ) if args.no_lock: diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index fa039ef..6d8dfba 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -74,3 +74,8 @@ script above runs on the Mac. dependent FFNs per `mx.eval`, as one decode step does. On the M4 Max it gives **TUNE**: 122-126 GB/s best (8-16 rows per threadgroup), 2.4-2.5x production's 50.5 GB/s, outputs within 6.4e-4. Quote only v2 rows. +- `bench_dsv4_mxfp8_residents.py` v1 also synced after every call. v2 + (`dsv4_mxfp8_resident_kill_test_v2`) chains 43 dependent calls per + `mx.eval`. On the M4 Max the verdict flips from KILL to **PASS** in two runs: + mxfp8 is 1.66x bf16 and takes 1.01-1.05x affine-8's time, still bit-exact. + The large tensors reach 350-450 GB/s. diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index 80b2b87..d521df6 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -99,3 +99,8 @@ or Mac-local files), plus 22 newly passing tests. dependent FFNs per `mx.eval`, as one decode step does. On the M4 Max it gives **TUNE**: 122-126 GB/s best (8-16 rows per threadgroup), 2.4-2.5x production's 50.5 GB/s, outputs within 6.4e-4. Quote only v2 rows. +- `bench_dsv4_mxfp8_residents.py` v1 also synced after every call. v2 + (`dsv4_mxfp8_resident_kill_test_v2`) chains 43 dependent calls per + `mx.eval`. On the M4 Max the verdict flips from KILL to **PASS** in two runs: + mxfp8 is 1.66x bf16 and takes 1.01-1.05x affine-8's time, still bit-exact. + The large tensors reach 350-450 GB/s. From 89e287dba5353b028295af4dd51949923827894e Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:50:53 -0800 Subject: [PATCH 13/74] docs: M4 Max reads 474-477 GB/s, not 409.4 The machine has the 40-core GPU (546 GB/s theoretical). Reductions over 1 GiB buffers read at 474-477 GB/s and a bf16 matvec at 458 GB/s. The plan's 409.4 GB/s estimates are therefore conservative. --- docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md | 4 ++-- .../2026-11-05-v41-plan-and-phase0-status.md | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index 6d8dfba..52f6a9c 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -1,7 +1,7 @@ # Phase 0 kill tests: toward DeepSeek-V4.1-Flash on one 128 GB Mac -Date: 2026-11-05. Target machine: the M4 Max (128 GB, 409.4 GB/s measured -MLX read). Phase 0 is the work that needs no V4.1 download and can be proven +Date: 2026-11-05. Target machine: the M4 Max (40-core GPU, 128 GB; 546 GB/s +theoretical, 474-477 GB/s measured MLX read on 2026-11-05). Phase 0 is the work that needs no V4.1 download and can be proven on the V4-Flash artifact. Every runtime change below is **opt-in**; defaults are unchanged until the kill test that guards it passes on the target Mac. diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index d521df6..3c577f3 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -1,7 +1,10 @@ # DeepSeek-V4.1-Flash on one 128 GB Mac: plan and Phase 0 status Date: 2026-11-05. Branch `jack/hopeful-cori-iszsfq`, draft PR jackmaxwil/keep#3. -Target machine: M4 Max, 128 GB, 409.4 GB/s measured MLX read. +Target machine: M4 Max, 40-core GPU, 128 GB. Memory bandwidth is 546 GB/s +theoretical; MLX reads 474-477 GB/s (1 GiB reductions) and a bf16 matvec +458 GB/s, measured 2026-11-05. The 409.4 GB/s used for the estimates below is +an older, lower measurement, so those estimates are conservative. ## The plan in brief From 7a6619d40609811161b11d4e0bea911505d4e700 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:10:45 -0800 Subject: [PATCH 14/74] ramp: tuned single-token E8P kernel for GPUs without NAX (opt-in) RAMP_E8P_M1=1 now dispatches to e8p_m1.metal instead of the generic M=1 kernel's E8P branch, for both gate/up (token layout) and down (per route). Each simdgroup owns several output rows, lanes load 4 or 8 codewords at once (whichever divides a scale group), and each threadgroup decodes the abs grid once into half2 pairs; signs are applied by XOR on the half2 bits. The decode is bit-identical to the reference on all 65,536 codewords. On the M4 Max the v2 kill test (43 chained FFNs per sync) now PASSES narrowly: 201.9-205.5 GB/s best over two runs, 3.97x production, up from 122-126 GB/s. A loads-only stub reaches ~410 GB/s, so decode ALU still takes half the time. The flag stays opt-in. The text-substituted per-route copy of gather_vqmm_m1.metal is gone; that file and the 8-bit M=1 path are unchanged. Tuning scripts and kernel variants are kept under the Phase 0 evidence directory. --- .../e8p-m1-synthetic-v2-tuned-run1.json | 63 ++++++++ .../e8p-m1-synthetic-v2-tuned-run2.json | 63 ++++++++ .../e8p-tuning/cpu.py | 23 +++ .../e8p-tuning/e8p_mr.metal | 87 +++++++++++ .../e8p-tuning/e8p_mr2.metal | 96 +++++++++++++ .../e8p-tuning/e8p_mr3.metal | 89 ++++++++++++ .../e8p-tuning/e8p_mr4.metal | 87 +++++++++++ .../e8p-tuning/e8p_mr5.metal | 96 +++++++++++++ .../e8p-tuning/e8p_mr6.metal | 98 +++++++++++++ .../e8p-tuning/e8p_mr7.metal | 100 +++++++++++++ .../e8p-tuning/e8p_stub.metal | 76 ++++++++++ .../e8p-tuning/h2h.py | 29 ++++ .../e8p-tuning/h2h3.py | 29 ++++ .../e8p-tuning/h2h4.py | 29 ++++ .../e8p-tuning/h2h5.py | 29 ++++ .../e8p-tuning/h2h67.py | 33 +++++ .../e8p-tuning/h2hs.py | 29 ++++ .../e8p-tuning/mr.py | 46 ++++++ .../e8p-tuning/mr2.py | 59 ++++++++ .../e8p-tuning/mr3.py | 46 ++++++ .../e8p-tuning/mr4.py | 46 ++++++ .../e8p-tuning/mr5.py | 46 ++++++ .../e8p-tuning/mr6.py | 46 ++++++ .../e8p-tuning/mr7.py | 46 ++++++ .../e8p-tuning/stub.py | 46 ++++++ benchmarks/bench_e8p_m1.py | 24 ++-- .../2026-11-05-phase0-kill-tests.md | 8 ++ .../2026-11-05-v41-plan-and-phase0-status.md | 8 ++ src/ramp/kernels/e8p_m1.metal | 103 +++++++++++++ src/ramp/kernels/e8p_m1.py | 135 ++++++++---------- tests/test_e8p_m1.py | 46 +++--- 31 files changed, 1650 insertions(+), 111 deletions(-) create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run1.json create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run2.json create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/cpu.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr2.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr3.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr4.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr5.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr6.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr7.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_stub.metal create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h3.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h4.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h5.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h67.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2hs.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr2.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr3.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr4.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr5.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr6.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr7.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/stub.py create mode 100644 src/ramp/kernels/e8p_m1.metal diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run1.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run1.json new file mode 100644 index 0000000..cb42a74 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run1.json @@ -0,0 +1,63 @@ +{ + "best": { + "effective_gb_per_s": 205.49661679791186, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.1865654072431891, + "rows_per_simdgroup": 4, + "simdgroups": 1, + "speedup_over_production": 3.969571859025884 + }, + "configs": [ + { + "effective_gb_per_s": 200.36558927025263, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.1913430351969721, + "rows_per_simdgroup": 8, + "simdgroups": 1, + "speedup_over_production": 3.870455957270122 + }, + { + "effective_gb_per_s": 205.49661679791186, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.1865654072431891, + "rows_per_simdgroup": 4, + "simdgroups": 1, + "speedup_over_production": 3.969571859025884 + }, + { + "effective_gb_per_s": 201.96111349692686, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.18983139544129807, + "rows_per_simdgroup": 4, + "simdgroups": 4, + "speedup_over_production": 3.901276649937917 + }, + { + "effective_gb_per_s": 201.68983785241895, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.19008672131539517, + "rows_per_simdgroup": 2, + "simdgroups": 4, + "speedup_over_production": 3.8960364266132834 + }, + { + "effective_gb_per_s": 192.8922985262998, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.19875630231433397, + "rows_per_simdgroup": 16, + "simdgroups": 1, + "speedup_over_production": 3.7260946286323495 + } + ], + "device": "Device(gpu, 0)", + "ffn_bytes": 38338560, + "ffns_per_sync": 43, + "mlx_version": "0.31.2", + "outputs_agree": true, + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "production_effective_gb_per_s": 51.76795485655721, + "production_median_ms_per_ffn": 0.7405847904602673, + "record_type": "ramp_e8p_m1_kill_test_v2", + "source": "synthetic", + "verdict": "PASS" +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run2.json b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run2.json new file mode 100644 index 0000000..5a09427 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-m1-synthetic-v2-tuned-run2.json @@ -0,0 +1,63 @@ +{ + "best": { + "effective_gb_per_s": 201.9003166348455, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.18988855807164812, + "rows_per_simdgroup": 4, + "simdgroups": 1, + "speedup_over_production": 3.9789912891068244 + }, + "configs": [ + { + "effective_gb_per_s": 196.16447165307085, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.1954408954736928, + "rows_per_simdgroup": 8, + "simdgroups": 1, + "speedup_over_production": 3.865950965057083 + }, + { + "effective_gb_per_s": 201.9003166348455, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.18988855807164812, + "rows_per_simdgroup": 4, + "simdgroups": 1, + "speedup_over_production": 3.9789912891068244 + }, + { + "effective_gb_per_s": 198.43019765127468, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.19320930208101175, + "rows_per_simdgroup": 4, + "simdgroups": 4, + "speedup_over_production": 3.9106032180134807 + }, + { + "effective_gb_per_s": 198.12068741620567, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.1935111395987617, + "rows_per_simdgroup": 2, + "simdgroups": 4, + "speedup_over_production": 3.904503482511547 + }, + { + "effective_gb_per_s": 189.32440694713122, + "max_relative_diff_vs_production": 0.000643500650767237, + "median_ms_per_ffn": 0.20250194160495127, + "rows_per_simdgroup": 16, + "simdgroups": 1, + "speedup_over_production": 3.7311490076580545 + } + ], + "device": "Device(gpu, 0)", + "ffn_bytes": 38338560, + "ffns_per_sync": 43, + "mlx_version": "0.31.2", + "outputs_agree": true, + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "production_effective_gb_per_s": 50.74158297043335, + "production_median_ms_per_ffn": 0.7555649184681432, + "record_type": "ramp_e8p_m1_kill_test_v2", + "source": "synthetic", + "verdict": "PASS" +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/cpu.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/cpu.py new file mode 100644 index 0000000..b81c57b --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/cpu.py @@ -0,0 +1,23 @@ +import sys, time +import numpy as np, mlx.core as mx +S = sys.argv[1] +src = open(f"{S}/mr.py").read(); src = src[: src.index("rng = np.random.default_rng(0)")] +ns = {}; sys.argv = [sys.argv[0], S]; exec(src, ns) +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +x = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +one = [mx.array(np.array([[i]], dtype=np.uint32)) for i in range(64)] +f = lambda r: ns["new"](x, gate, r, 0, 4, 1, 8, 1) +for _ in range(3): mx.eval([f(r) for r in routes * 4]) +for label, rs in (("6 routes", routes), ("1 route ", one)): + best_build = best_total = 1e9 + for _ in range(20): + a = time.perf_counter(); outs = [f(r) for r in rs * 4]; b = time.perf_counter(); mx.eval(outs); c = time.perf_counter() + best_build = min(best_build, (b - a) / 256); best_total = min(best_total, (c - a) / 256) + print(f"{label}: build {best_build*1e6:.1f} us/call, build+eval {best_total*1e6:.1f} us/call") +# matmul build overhead for comparison +w = mx.random.normal((1560, 4096)).astype(mx.bfloat16); xb = x.astype(mx.bfloat16); mx.eval(w, xb) +a = time.perf_counter(); outs = [xb @ w.T for _ in range(256)]; b = time.perf_counter(); mx.eval(outs); c = time.perf_counter() +print(f"bf16 matmul: build {(b-a)/256*1e6:.1f} us/call, build+eval {(c-a)/256*1e6:.1f} us/call") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr.metal new file mode 100644 index 0000000..f5ccbd2 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr.metal @@ -0,0 +1,87 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint local_codebook[256]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float xv[CHUNK * 8]; + for (uint i = 0; i < uint(CHUNK) * 8u; ++i) { + xv[i] = float(x[x_row + cw0 * 8u + i]); + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint abs_code = uint(TG_CODEBOOK) != 0u ? local_codebook[codeword >> 8u] : codebook[codeword >> 8u]; + float offset = parity != 0u ? -0.25f : 0.25f; + const uint shuffle[8] = {0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u}; + for (uint d = 0; d < 8u; ++d) { + uint p = shuffle[d]; + float v = (float((abs_code >> (4u * p)) & 0xFu) - 8.0f) * 0.5f; + v = ((effective >> p) & 1u) != 0u ? -v : v; + part += (v + offset) * xv[i * 8u + d]; + } + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr2.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr2.metal new file mode 100644 index 0000000..690ab5c --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr2.metal @@ -0,0 +1,96 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +const device half4* abs4 = (const device half4*)abs_tab; +const device ushort4* sign4 = (const device ushort4*)sign_tab; +threadgroup half4 tg_abs[512]; +threadgroup ushort4 tg_sign[512]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 512u; idx += threads_per_threadgroup.x) { + tg_abs[idx] = abs4[idx]; + tg_sign[idx] = sign4[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float4 xe[CHUNK * 2]; + float xs[CHUNK]; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint xb = x_row + (cw0 + i) * 8u; + xe[2 * i] = float4(float(x[xb]), float(x[xb + 1u]), float(x[xb + 2u]), float(x[xb + 3u])); + xe[2 * i + 1] = float4(float(x[xb + 4u]), float(x[xb + 5u]), float(x[xb + 6u]), float(x[xb + 7u])); + xs[i] = dot(xe[2 * i] + xe[2 * i + 1], float4(1.0f)); + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint a = (codeword >> 8u) * 2u; + uint sg = (codeword & 0xFFu) * 2u; + half4 a0, a1; + ushort4 s0, s1; + if (uint(TG_CODEBOOK) != 0u) { + a0 = tg_abs[a]; a1 = tg_abs[a + 1u]; s0 = tg_sign[sg]; s1 = tg_sign[sg + 1u]; + } else { + a0 = abs4[a]; a1 = abs4[a + 1u]; s0 = sign4[sg]; s1 = sign4[sg + 1u]; + } + float4 w0 = float4(as_type(as_type(a0) ^ s0)); + float4 w1 = float4(as_type(as_type(a1) ^ s1)); + float offset = (popcount(codeword & 0xFFu) & 1u) != 0u ? -0.25f : 0.25f; + part += dot(w0, xe[2 * i]) + dot(w1, xe[2 * i + 1]) + offset * xs[i]; + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr3.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr3.metal new file mode 100644 index 0000000..3b50a31 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr3.metal @@ -0,0 +1,89 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint local_codebook[256]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float xv[CHUNK * 8]; + for (uint i = 0; i < uint(CHUNK) * 8u; ++i) { + xv[i] = float(x[x_row + cw0 * 8u + i]); + } + uint group = cw0 / codewords_per_group; + // Issue every row's code and scale load before any decode. + uint4 words[ROWS_PER_SG]; + float row_scale[ROWS_PER_SG]; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = min(row0 + r, out_dim - 1u); + uint base = expert * out_dim + row; + if (uint(CHUNK) == 8u) { + words[r] = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words[r] = uint4(((const device uint2*)(codes + base * codewords))[cw0 / 4u], 0u, 0u); + } + row_scale[r] = float(scales[base * groups + group]); + } + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[r][i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint abs_code = uint(TG_CODEBOOK) != 0u ? local_codebook[codeword >> 8u] : codebook[codeword >> 8u]; + float offset = parity != 0u ? -0.25f : 0.25f; + const uint shuffle[8] = {0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u}; + for (uint d = 0; d < 8u; ++d) { + uint p = shuffle[d]; + float v = (float((abs_code >> (4u * p)) & 0xFu) - 8.0f) * 0.5f; + v = ((effective >> p) & 1u) != 0u ? -v : v; + part += (v + offset) * xv[i * 8u + d]; + } + } + acc[r] += part * row_scale[r]; + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr4.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr4.metal new file mode 100644 index 0000000..f8f1291 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr4.metal @@ -0,0 +1,87 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint local_codebook[256]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + X_T xv[CHUNK * 8]; + for (uint i = 0; i < uint(CHUNK) * 8u; ++i) { + xv[i] = x[x_row + cw0 * 8u + i]; + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint abs_code = uint(TG_CODEBOOK) != 0u ? local_codebook[codeword >> 8u] : codebook[codeword >> 8u]; + float offset = parity != 0u ? -0.25f : 0.25f; + const uint shuffle[8] = {0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u}; + for (uint d = 0; d < 8u; ++d) { + uint p = shuffle[d]; + float v = (float((abs_code >> (4u * p)) & 0xFu) - 8.0f) * 0.5f; + v = ((effective >> p) & 1u) != 0u ? -v : v; + part += (v + offset) * float(xv[i * 8u + d]); + } + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr5.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr5.metal new file mode 100644 index 0000000..46b9020 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr5.metal @@ -0,0 +1,96 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint local_codebook[256]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float2 xv[CHUNK * 4]; + float xs[CHUNK]; + for (uint i = 0; i < uint(CHUNK); ++i) { + float sum = 0.0f; + for (uint k = 0; k < 4u; ++k) { + uint xi = x_row + (cw0 + i) * 8u + 2u * k; + xv[i * 4u + k] = float2(float(x[xi]), float(x[xi + 1u])); + sum += xv[i * 4u + k].x + xv[i * 4u + k].y; + } + xs[i] = sum; + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint abs_code = uint(TG_CODEBOOK) != 0u ? local_codebook[codeword >> 8u] : codebook[codeword >> 8u]; + // Nibbles k and k+4 are dims 2k and 2k+1: one half2 per pair. + // (1024 + n) * 0.5 - 516 = (n - 8) / 2, exact in half. + uint spread = (effective & 0x0Fu) | ((effective & 0xF0u) << 12u); + for (uint k = 0; k < 4u; ++k) { + half2 h = as_type(((abs_code >> (4u * k)) & 0x000F000Fu) | 0x64006400u); + half2 v = fma(h, half2(0.5h), half2(-516.0h)); + v = as_type(as_type(v) ^ ((spread << (15u - k)) & 0x80008000u)); + part += dot(float2(v), xv[i * 4u + k]); + } + part += (parity != 0u ? -0.25f : 0.25f) * xs[i]; + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr6.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr6.metal new file mode 100644 index 0000000..b42aafc --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr6.metal @@ -0,0 +1,98 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint4 decoded[256]; +for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + uint c = codebook[idx]; + uint4 entry; + for (uint k = 0; k < 4u; ++k) { + half2 h = as_type(((c >> (4u * k)) & 0x000F000Fu) | 0x64006400u); + entry[k] = as_type(fma(h, half2(0.5h), half2(-516.0h))); + } + decoded[idx] = entry; +} +threadgroup_barrier(mem_flags::mem_threadgroup); + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float2 xv[CHUNK * 4]; + float xs[CHUNK]; + for (uint i = 0; i < uint(CHUNK); ++i) { + float sum = 0.0f; + for (uint k = 0; k < 4u; ++k) { + uint xi = x_row + (cw0 + i) * 8u + 2u * k; + xv[i * 4u + k] = float2(float(x[xi]), float(x[xi + 1u])); + sum += xv[i * 4u + k].x + xv[i * 4u + k].y; + } + xs[i] = sum; + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint4 entry = decoded[codeword >> 8u]; + // Nibbles k and k+4 are dims 2k and 2k+1: one half2 per pair. + // (1024 + n) * 0.5 - 516 = (n - 8) / 2, exact in half. + uint spread = (effective & 0x0Fu) | ((effective & 0xF0u) << 12u); + for (uint k = 0; k < 4u; ++k) { + half2 v = as_type(entry[k] ^ ((spread << (15u - k)) & 0x80008000u)); + part += dot(float2(v), xv[i * 4u + k]); + } + part += (parity != 0u ? -0.25f : 0.25f) * xs[i]; + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr7.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr7.metal new file mode 100644 index 0000000..6bc0763 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_mr7.metal @@ -0,0 +1,100 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint local_codebook[256]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float2 xv[CHUNK * 4]; + half2 xh[CHUNK * 4]; + float xs[CHUNK]; + for (uint i = 0; i < uint(CHUNK); ++i) { + float sum = 0.0f; + for (uint k = 0; k < 4u; ++k) { + uint xi = x_row + (cw0 + i) * 8u + 2u * k; + xv[i * 4u + k] = float2(float(x[xi]), float(x[xi + 1u])); + xh[i * 4u + k] = half2(xv[i * 4u + k]); + sum += xv[i * 4u + k].x + xv[i * 4u + k].y; + } + xs[i] = sum; + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint abs_code = uint(TG_CODEBOOK) != 0u ? local_codebook[codeword >> 8u] : codebook[codeword >> 8u]; + // Nibbles k and k+4 are dims 2k and 2k+1: one half2 per pair. + // (1024 + n) * 0.5 - 516 = (n - 8) / 2, exact in half. + uint spread = (effective & 0x0Fu) | ((effective & 0xF0u) << 12u); + half2 hacc = half2(0.0h); + for (uint k = 0; k < 4u; ++k) { + half2 h = as_type(((abs_code >> (4u * k)) & 0x000F000Fu) | 0x64006400u); + half2 v = fma(h, half2(0.5h), half2(-516.0h)); + v = as_type(as_type(v) ^ ((spread << (15u - k)) & 0x80008000u)); + hacc = fma(v, xh[i * 4u + k], hacc); + } + part += float(hacc.x) + float(hacc.y); + part += (parity != 0u ? -0.25f : 0.25f) * xs[i]; + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_stub.metal b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_stub.metal new file mode 100644 index 0000000..643f7e5 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/e8p_stub.metal @@ -0,0 +1,76 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK consecutive +// codewords (one 8- or 16-byte load per row), so a simdgroup reads one +// contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// CHUNK * 8 activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d]; bit p of the effective signs +// negates nibble p, and every value gets +-0.25 by parity. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +threadgroup uint local_codebook[256]; +if (uint(TG_CODEBOOK) != 0u) { + for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + local_codebook[idx] = codebook[idx]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float xv[CHUNK * 8]; + for (uint i = 0; i < uint(CHUNK) * 8u; ++i) { + xv[i] = float(x[x_row + cw0 * 8u + i]); + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + part += float(codeword) * xv[i * 8u]; + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h.py new file mode 100644 index 0000000..7693e61 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h.py @@ -0,0 +1,29 @@ +import sys, importlib.util, time +import numpy as np, mlx.core as mx +S = sys.argv[1]; sys.argv = [sys.argv[0], S] +def load(name): + src = open(f"{S}/{name}.py").read() + src = src[: src.index("rng = np.random.default_rng(0)")] # defs only + ns = {}; exec(src, ns); return ns +A, B = load("mr"), load("mr2") +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)); xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +cfgA = [(4, 1, 8, 1), (8, 1, 8, 0), (2, 1, 8, 0), (8, 2, 8, 0)] +cfgB = [(2, 2, 4, 1), (8, 2, 8, 1), (4, 1, 4, 0), (4, 1, 8, 1)] +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nb = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases = {"old": lambda r: A["old"](x, L, r, pr)} + for c in cfgA: cases[f"arith {c}"] = lambda r, c=c: A["new"](x, L, r, pr, *c) + for c in cfgB: cases[f"table {c}"] = lambda r, c=c: B["new"](x, L, r, pr, *c) + for f in cases.values(): once(f); once(f) + best = {k: 1e9 for k in cases} + for _ in range(25): + for k, f in cases.items(): best[k] = min(best[k], once(f)) + for k in sorted(best, key=best.get): + print(f"{name} {k:22s} {best[k]*1e3:.4f} ms {nb / best[k] / 1e9:6.1f} GB/s x{best['old'] / best[k]:.2f}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h3.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h3.py new file mode 100644 index 0000000..5dc6fe4 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h3.py @@ -0,0 +1,29 @@ +import sys, importlib.util, time +import numpy as np, mlx.core as mx +S = sys.argv[1]; sys.argv = [sys.argv[0], S] +def load(name): + src = open(f"{S}/{name}.py").read() + src = src[: src.index("rng = np.random.default_rng(0)")] # defs only + ns = {}; exec(src, ns); return ns +A, B = load("mr"), load("mr3") +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)); xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +cfgA = [(4, 1, 8, 1), (8, 1, 8, 0)] +cfgB = [(4, 1, 8, 1), (8, 1, 8, 0), (8, 2, 8, 0), (16, 1, 8, 0), (16, 2, 8, 1)] +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nb = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases = {"old": lambda r: A["old"](x, L, r, pr)} + for c in cfgA: cases[f"arith {c}"] = lambda r, c=c: A["new"](x, L, r, pr, *c) + for c in cfgB: cases[f"pre {c}"] = lambda r, c=c: B["new"](x, L, r, pr, *c) + for f in cases.values(): once(f); once(f) + best = {k: 1e9 for k in cases} + for _ in range(25): + for k, f in cases.items(): best[k] = min(best[k], once(f)) + for k in sorted(best, key=best.get): + print(f"{name} {k:22s} {best[k]*1e3:.4f} ms {nb / best[k] / 1e9:6.1f} GB/s x{best['old'] / best[k]:.2f}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h4.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h4.py new file mode 100644 index 0000000..ce9645f --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h4.py @@ -0,0 +1,29 @@ +import sys, importlib.util, time +import numpy as np, mlx.core as mx +S = sys.argv[1]; sys.argv = [sys.argv[0], S] +def load(name): + src = open(f"{S}/{name}.py").read() + src = src[: src.index("rng = np.random.default_rng(0)")] # defs only + ns = {}; exec(src, ns); return ns +A, B = load("mr"), load("mr4") +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)); xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +cfgA = [(4, 1, 8, 1), (8, 1, 8, 0)] +cfgB = [(4, 1, 8, 1), (8, 1, 8, 0), (8, 2, 8, 1), (16, 1, 8, 1), (4, 2, 8, 1), (2, 1, 8, 1)] +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nb = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases = {"old": lambda r: A["old"](x, L, r, pr)} + for c in cfgA: cases[f"arith {c}"] = lambda r, c=c: A["new"](x, L, r, pr, *c) + for c in cfgB: cases[f"xhalf {c}"] = lambda r, c=c: B["new"](x, L, r, pr, *c) + for f in cases.values(): once(f); once(f) + best = {k: 1e9 for k in cases} + for _ in range(25): + for k, f in cases.items(): best[k] = min(best[k], once(f)) + for k in sorted(best, key=best.get): + print(f"{name} {k:22s} {best[k]*1e3:.4f} ms {nb / best[k] / 1e9:6.1f} GB/s x{best['old'] / best[k]:.2f}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h5.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h5.py new file mode 100644 index 0000000..3322616 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h5.py @@ -0,0 +1,29 @@ +import sys, importlib.util, time +import numpy as np, mlx.core as mx +S = sys.argv[1]; sys.argv = [sys.argv[0], S] +def load(name): + src = open(f"{S}/{name}.py").read() + src = src[: src.index("rng = np.random.default_rng(0)")] # defs only + ns = {}; exec(src, ns); return ns +A, B = load("mr"), load("mr5") +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)); xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +cfgA = [(4, 1, 8, 1), (8, 1, 8, 0)] +cfgB = [(4, 1, 8, 1), (8, 1, 8, 0), (4, 4, 8, 1), (2, 1, 8, 1), (8, 2, 8, 1), (4, 1, 4, 1)] +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nb = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases = {"old": lambda r: A["old"](x, L, r, pr)} + for c in cfgA: cases[f"arith {c}"] = lambda r, c=c: A["new"](x, L, r, pr, *c) + for c in cfgB: cases[f"half2 {c}"] = lambda r, c=c: B["new"](x, L, r, pr, *c) + for f in cases.values(): once(f); once(f) + best = {k: 1e9 for k in cases} + for _ in range(25): + for k, f in cases.items(): best[k] = min(best[k], once(f)) + for k in sorted(best, key=best.get): + print(f"{name} {k:22s} {best[k]*1e3:.4f} ms {nb / best[k] / 1e9:6.1f} GB/s x{best['old'] / best[k]:.2f}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h67.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h67.py new file mode 100644 index 0000000..2e197d0 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2h67.py @@ -0,0 +1,33 @@ +import sys, importlib.util, time +import numpy as np, mlx.core as mx +S = sys.argv[1]; sys.argv = [sys.argv[0], S] +def load(name): + src = open(f"{S}/{name}.py").read() + src = src[: src.index("rng = np.random.default_rng(0)")] # defs only + ns = {}; exec(src, ns); return ns +A, B, C, D = load("mr"), load("mr5"), load("mr6"), load("mr7") +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)); xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +cfgA = [(4, 1, 8, 1), (8, 1, 8, 0), (2, 1, 8, 0), (8, 2, 8, 0)] +cfgB = [(2, 2, 4, 1), (8, 2, 8, 1), (4, 1, 4, 0), (4, 1, 8, 1)] +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nb = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases = {"old": lambda r: A["old"](x, L, r, pr)} + for c in [(4, 4, 8, 1), (8, 1, 8, 0), (4, 1, 8, 1), (8, 4, 8, 1), (2, 4, 8, 1)]: + cases[f"h2 {c}"] = lambda r, c=c: B["new"](x, L, r, pr, *c) + cases[f"tab {c}"] = lambda r, c=c: C["new"](x, L, r, pr, *c) + cases[f"hacc {c}"] = lambda r, c=c: D["new"](x, L, r, pr, *c) + for f in cases.values(): once(f); once(f) + best = {k: 1e9 for k in cases} + for _ in range(25): + for k, f in cases.items(): best[k] = min(best[k], once(f)) + ref = np.array(cases["old"](routes[0]).astype(mx.float32)).reshape(-1) + for k in sorted(best, key=best.get)[:9] + ["old"]: + d = np.max(np.abs(np.array(cases[k](routes[0]).astype(mx.float32)).reshape(-1) - ref)) / np.max(np.abs(ref)) + print(f"{name} {k:22s} {best[k]*1e3:.4f} ms {nb / best[k] / 1e9:6.1f} GB/s x{best['old'] / best[k]:.2f} diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2hs.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2hs.py new file mode 100644 index 0000000..1a30a15 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/h2hs.py @@ -0,0 +1,29 @@ +import sys, importlib.util, time +import numpy as np, mlx.core as mx +S = sys.argv[1]; sys.argv = [sys.argv[0], S] +def load(name): + src = open(f"{S}/{name}.py").read() + src = src[: src.index("rng = np.random.default_rng(0)")] # defs only + ns = {}; exec(src, ns); return ns +A, B = load("mr"), load("stub") +from benchmarks.bench_e8p_m1 import synthetic_projection +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)); xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +cfgA = [(4, 1, 8, 1), (8, 1, 8, 0)] +cfgB = [(4, 1, 8, 1), (8, 1, 8, 0), (1, 1, 8, 1), (4, 4, 8, 1)] +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nb = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases = {"old": lambda r: A["old"](x, L, r, pr)} + for c in cfgA: cases[f"arith {c}"] = lambda r, c=c: A["new"](x, L, r, pr, *c) + for c in cfgB: cases[f"stub {c}"] = lambda r, c=c: B["new"](x, L, r, pr, *c) + for f in cases.values(): once(f); once(f) + best = {k: 1e9 for k in cases} + for _ in range(25): + for k, f in cases.items(): best[k] = min(best[k], once(f)) + for k in sorted(best, key=best.get): + print(f"{name} {k:22s} {best[k]*1e3:.4f} ms {nb / best[k] / 1e9:6.1f} GB/s x{best['old'] / best[k]:.2f}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr.py new file mode 100644 index 0000000..23ac27a --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr2.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr2.py new file mode 100644 index 0000000..8cf83fa --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr2.py @@ -0,0 +1,59 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +SHUF = np.array([0, 4, 1, 5, 2, 6, 3, 7]) +_T = {} +def tables(cb): + if id(cb) not in _T: + c = np.array(cb).astype(np.uint32) + nib = (c[:, None] >> (4 * SHUF[None, :]).astype(np.uint32)) & 0xF + abs_tab = ((nib.astype(np.float32) - 8) * 0.5).astype(np.float16) + s = np.arange(256, dtype=np.uint32) + par = np.array([bin(v).count("1") & 1 for v in s], dtype=np.uint32) + eff = s ^ par + sign_tab = (((eff[:, None] >> SHUF[None, :].astype(np.uint32)) & 1) << 15).astype(np.uint16) + _T[id(cb)] = (mx.array(abs_tab), mx.array(sign_tab)) + return _T[id(cb)] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "abs_tab", "sign_tab", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr2.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, *tables(L.codebook), r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr3.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr3.py new file mode 100644 index 0000000..c6aa7ec --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr3.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr3.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr4.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr4.py new file mode 100644 index 0000000..ad8d317 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr4.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr4.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("X_T", x.dtype), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr5.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr5.py new file mode 100644 index 0000000..eafd7b2 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr5.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr5.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr6.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr6.py new file mode 100644 index 0000000..7a88f09 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr6.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr6.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr7.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr7.py new file mode 100644 index 0000000..b93620b --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/mr7.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_mr7.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/stub.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/stub.py new file mode 100644 index 0000000..329f59c --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-tuning/stub.py @@ -0,0 +1,46 @@ +import sys, time, itertools +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.kernels.gather_vqmm import _qmv_output_dtype +S = sys.argv[1] +K = mx.fast.metal_kernel(name="e8p_mr", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], source=open(S + "/e8p_stub.metal").read()) +def new(x, L, r, per_route, R, SG, C, tg): + routes = r.size; out_dim = L.codes.shape[1] + (o,) = K(inputs=[x, L.codes, L.scales, L.codebook, r.reshape(1, routes)], + template=[("ROWS_PER_SG", R), ("SIMDGROUPS", SG), ("CHUNK", C), ("TG_CODEBOOK", tg), + ("PER_ROUTE", per_route), ("OUT_T", _qmv_output_dtype(x.dtype))], + grid=(32 * SG, (out_dim + R * SG - 1) // (R * SG), routes), threadgroup=(32 * SG, 1, 1), + output_shapes=[(1, routes, out_dim)], output_dtypes=[_qmv_output_dtype(x.dtype)]) + return o +def old(x, L, r, per_route): + f = e8p_m1.e8p_m1_per_route if per_route else e8p_m1.e8p_m1_token_layout + return f(x, L.codes, L.scales, L.codebook, r, output_dims=L.codes.shape[1], rows_per_threadgroup=16) +rng = np.random.default_rng(0) +gate = synthetic_projection(experts=64, out=2048, inp=4096, group_size=512, seed=1) +down = synthetic_projection(experts=64, out=4096, inp=2048, group_size=512, seed=3) +routes = [mx.array(np.sort(rng.choice(64, 6, replace=False))[None].astype(np.uint32)) for _ in range(64)] +xg = mx.array(rng.normal(size=(1, 4096)).astype(np.float16)) +xd = mx.array(rng.normal(size=(6, 2048)).astype(np.float16)) +def once(fn): + a = time.perf_counter(); mx.eval([fn(r) for r in routes * 4]); return (time.perf_counter() - a) / 256 +cases = {} +for name, L, x, pr in (("gate", gate, xg, 0), ("down", down, xd, 1)): + nbytes = 6 * (L.codes.nbytes + L.scales.nbytes) // 64 + cases[(name, "old")] = (lambda r, L=L, x=x, pr=pr: old(x, L, r, pr), nbytes, L, x, pr) + for R, SG, C, tg in itertools.product((1, 2, 4, 8), (1, 2, 4), (4, 8), (1, 0)): + cases[(name, f"R{R} SG{SG} C{C} tg{tg}")] = (lambda r, L=L, x=x, pr=pr, a=(R, SG, C, tg): new(x, L, r, pr, *a), nbytes, L, x, pr) +for k, (fn, *_ ) in cases.items(): + for _ in range(2): once(fn) +best = {k: 1e9 for k in cases} +for _ in range(8): + for k, (fn, *_ ) in cases.items(): best[k] = min(best[k], once(fn)) +for name in ("gate", "down"): + ref = np.array(cases[(name, "old")][0](routes[0]).astype(mx.float32)).reshape(-1) + rows = sorted(((k[1], best[k]) for k in cases if k[0] == name), key=lambda t: t[1]) + for cfg, t in rows[:8] + [r for r in rows if r[0] == "old"]: + got = np.array(cases[(name, cfg)][0](routes[0]).astype(mx.float32)).reshape(-1) + d = np.max(np.abs(got - ref)) / np.max(np.abs(ref)) + print(f"{name} {cfg:18s} {t*1e3:.4f} ms {cases[(name, cfg)][1] / t / 1e9:6.1f} GB/s diff {d:.1e}") diff --git a/benchmarks/bench_e8p_m1.py b/benchmarks/bench_e8p_m1.py index 278a030..47faae7 100644 --- a/benchmarks/bench_e8p_m1.py +++ b/benchmarks/bench_e8p_m1.py @@ -4,8 +4,8 @@ ways in the same process: - ``production``: today's dispatch (the scalar kernel on M1-M4; NAX on M5); -- ``e8p_m1``: ``RAMP_E8P_M1=1``, the M=1 E8P kernels, swept over row packing - and threadgroup codebook staging. +- ``e8p_m1``: ``RAMP_E8P_M1=1``, the M=1 E8P kernel, swept over rows per + simdgroup and simdgroups per threadgroup. Effective bandwidth is the expert bytes one FFN reads (6 experts x 3 projections of 2-bit codes plus their scales) over the median time per FFN. @@ -130,15 +130,15 @@ def run( import ramp.nn.switch_linear as switch_linear try: - for rows_per_tg, tg_codebook in sweep: + for rows_per_sg, simdgroups in sweep: switch_linear.e8p_m1_token_layout = ( - lambda *a, _r=rows_per_tg, _c=tg_codebook, **k: original_token( - *a, rows_per_threadgroup=_r, use_threadgroup_codebook=_c, **k + lambda *a, _r=rows_per_sg, _s=simdgroups, **k: original_token( + *a, rows_per_simdgroup=_r, simdgroups=_s, **k ) ) switch_linear.e8p_m1_per_route = ( - lambda *a, _r=rows_per_tg, _c=tg_codebook, **k: original_route( - *a, rows_per_threadgroup=_r, use_threadgroup_codebook=_c, **k + lambda *a, _r=rows_per_sg, _s=simdgroups, **k: original_route( + *a, rows_per_simdgroup=_r, simdgroups=_s, **k ) ) got = [np.array(_ffn(layers, x, r, intermediate).astype(mx.float32)) for r in routes_list[:4]] @@ -149,8 +149,8 @@ def run( ms = _median_ms(layers, x, routes_list, intermediate, iterations, warmup, ffns_per_sync) configs.append( { - "rows_per_threadgroup": rows_per_tg, - "threadgroup_codebook": tg_codebook, + "rows_per_simdgroup": rows_per_sg, + "simdgroups": simdgroups, "median_ms_per_ffn": ms, "effective_gb_per_s": ffn_bytes / (ms / 1e3) / 1e9, "speedup_over_production": production_ms / ms, @@ -191,7 +191,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--ffns-per-sync", type=int, default=43, help="chained FFNs per mx.eval (V4-Flash has 43 layers)") parser.add_argument("--warmup", type=int, default=20) parser.add_argument("--seed", type=int, default=20261105) - parser.add_argument("--sweep", default="8:1,16:1,32:1,64:1,32:0", help="rows_per_threadgroup:tg_codebook,...") + parser.add_argument("--sweep", default="4:1,8:1,4:4,2:4,16:1", help="rows_per_simdgroup:simdgroups,...") parser.add_argument("--output-json", type=Path) args = parser.parse_args(argv) @@ -215,8 +215,8 @@ def main(argv: Sequence[str] | None = None) -> int: } sweep = [] for item in args.sweep.split(","): - rows, codebook = item.split(":") - sweep.append((int(rows), codebook.strip() == "1")) + rows, simdgroups = item.split(":") + sweep.append((int(rows), int(simdgroups))) result = { "record_type": "ramp_e8p_m1_kill_test_v2", "device": str(mx.default_device()), diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index 52f6a9c..d73b9d4 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -74,6 +74,14 @@ script above runs on the Mac. dependent FFNs per `mx.eval`, as one decode step does. On the M4 Max it gives **TUNE**: 122-126 GB/s best (8-16 rows per threadgroup), 2.4-2.5x production's 50.5 GB/s, outputs within 6.4e-4. Quote only v2 rows. +- The tuned kernel (`src/ramp/kernels/e8p_m1.metal`, 2026-11-05) then + **PASSES** v2, narrowly: 201.9-205.5 GB/s best (4 rows per simdgroup), 3.97x + production, outputs unchanged at 6.4e-4. Each simdgroup owns several rows, + lanes load 8 codewords at once, and the abs grid is decoded once per + threadgroup into half2 pairs, with signs applied by XOR. A loads-only stub + of the same kernel reads at ~410 GB/s, so decode ALU is still half the + time. Not yet the default: it clears the bar by 1-3% while other jobs were + loading the machine. - `bench_dsv4_mxfp8_residents.py` v1 also synced after every call. v2 (`dsv4_mxfp8_resident_kill_test_v2`) chains 43 dependent calls per `mx.eval`. On the M4 Max the verdict flips from KILL to **PASS** in two runs: diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index 3c577f3..ec9991c 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -102,6 +102,14 @@ or Mac-local files), plus 22 newly passing tests. dependent FFNs per `mx.eval`, as one decode step does. On the M4 Max it gives **TUNE**: 122-126 GB/s best (8-16 rows per threadgroup), 2.4-2.5x production's 50.5 GB/s, outputs within 6.4e-4. Quote only v2 rows. +- The tuned kernel (`src/ramp/kernels/e8p_m1.metal`, 2026-11-05) then + **PASSES** v2, narrowly: 201.9-205.5 GB/s best (4 rows per simdgroup), 3.97x + production, outputs unchanged at 6.4e-4. Each simdgroup owns several rows, + lanes load 8 codewords at once, and the abs grid is decoded once per + threadgroup into half2 pairs, with signs applied by XOR. A loads-only stub + of the same kernel reads at ~410 GB/s, so decode ALU is still half the + time. Not yet the default: it clears the bar by 1-3% while other jobs were + loading the machine. - `bench_dsv4_mxfp8_residents.py` v1 also synced after every call. v2 (`dsv4_mxfp8_resident_kill_test_v2`) chains 43 dependent calls per `mx.eval`. On the M4 Max the verdict flips from KILL to **PASS** in two runs: diff --git a/src/ramp/kernels/e8p_m1.metal b/src/ramp/kernels/e8p_m1.metal new file mode 100644 index 0000000..96b55d5 --- /dev/null +++ b/src/ramp/kernels/e8p_m1.metal @@ -0,0 +1,103 @@ +// Single-token (M=1) gather matvec for 16-bit E8P codes on GPUs without NAX. +// +// Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK (4 or 8) +// consecutive codewords: one 8- or 16-byte load per row, so a simdgroup reads +// a contiguous 32 * CHUNK * 2-byte span per row per step, and the lane's +// activations are loaded once and reused for every row. +// +// E8P layout (see mlx_vq_decode_e8p_value in common_e8.metal): codeword bits +// 0-7 are signs, bits 8-15 index the 256-entry abs grid of packed nibbles. +// Dimension d reads nibble {0,4,1,5,2,6,3,7}[d], so nibbles k and k+4 are +// dims 2k and 2k+1 and decode together as one half2. Bit p of the effective +// signs negates nibble p, and every value gets +-0.25 by parity. +// +// Decode is the bottleneck (loads alone run at ~410 GB/s on an M4 Max), so +// each threadgroup first decodes the abs grid into half2 pairs, and a +// codeword then costs one threadgroup read plus one XOR per pair for signs. +// +// PER_ROUTE=0: one activation row for every route (gate/up). +// PER_ROUTE=1: route r reads activation row r (down). +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint top_k = uint(rhs_indices_shape[1]); +uint expert = uint(rhs_indices[route % top_k]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; + +// (1024 + n) * 0.5 - 516 = (n - 8) / 2, exact in half. +threadgroup uint4 decoded[256]; +for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + uint packed = codebook[idx]; + uint4 entry; + for (uint k = 0; k < 4u; ++k) { + half2 h = as_type(((packed >> (4u * k)) & 0x000F000Fu) | 0x64006400u); + entry[k] = as_type(fma(h, half2(0.5h), half2(-516.0h))); + } + decoded[idx] = entry; +} +threadgroup_barrier(mem_flags::mem_threadgroup); + +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float2 xv[CHUNK * 4]; + float xs[CHUNK]; + for (uint i = 0; i < uint(CHUNK); ++i) { + float sum = 0.0f; + for (uint k = 0; k < 4u; ++k) { + uint xi = x_row + (cw0 + i) * 8u + 2u * k; + xv[i * 4u + k] = float2(float(x[xi]), float(x[xi + 1u])); + sum += xv[i * 4u + k].x + xv[i * 4u + k].y; + } + xs[i] = sum; + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint4 words; + if (uint(CHUNK) == 8u) { + words = ((const device uint4*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.xy = ((const device uint2*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint codeword = (words[i >> 1u] >> ((i & 1u) * 16u)) & 0xFFFFu; + uint signs = codeword & 0xFFu; + uint parity = popcount(signs) & 1u; + uint effective = signs ^ parity; + uint4 entry = decoded[codeword >> 8u]; + // Sign bits k and k+4 move to bits 15 and 31 of pair k. + uint spread = (effective & 0x0Fu) | ((effective & 0xF0u) << 12u); + for (uint k = 0; k < 4u; ++k) { + half2 v = as_type(entry[k] ^ ((spread << (15u - k)) & 0x80008000u)); + part += dot(float2(v), xv[i * 4u + k]); + } + part += (parity != 0u ? -0.25f : 0.25f) * xs[i]; + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/src/ramp/kernels/e8p_m1.py b/src/ramp/kernels/e8p_m1.py index cf4fb97..c54c29b 100644 --- a/src/ramp/kernels/e8p_m1.py +++ b/src/ramp/kernels/e8p_m1.py @@ -8,18 +8,17 @@ E8P branch; nothing dispatched to it. With ``RAMP_E8P_M1=1`` (opt-in until ``benchmarks/bench_e8p_m1.py`` passes on -the target Mac), M=1 calls with 16-bit codes use: +the target Mac), M=1 calls with 16-bit codes use ``e8p_m1.metal``: - :func:`e8p_m1_token_layout` for gate/up (one activation row, top-k - experts): the existing M=1 kernel with ``CODE_BITS=16``. -- :func:`e8p_m1_per_route` for down (one activation row *per route*): the - same kernel source with the activation read offset by route. It is built as - a separate kernel from a text substitution, so the production 8-bit kernel - stays byte-identical. - -The E8P branch decodes each codeword once per dimension, but the compiler lifts -the per-codeword work out of that loop: a decode-once rewrite measured slower -on the M4 Max (113 vs 154 GB/s, 2026-11-05). + experts). +- :func:`e8p_m1_per_route` for down (one activation row *per route*). + +The kernel gives each simdgroup several rows, loads 4-8 codewords per lane at +once, and decodes the abs grid once per threadgroup. On the M4 Max that is +1.6-2.0x the generic M=1 kernel's E8P branch, which it replaces here; the +production 8-bit M=1 kernel is untouched. A loads-only stub reaches ~410 GB/s, +so decode ALU is still the limit. """ from __future__ import annotations @@ -33,7 +32,6 @@ "e8p_m1_enabled", "e8p_m1_per_route", "e8p_m1_token_layout", - "per_route_x_source", "should_use_e8p_m1", ] @@ -41,10 +39,6 @@ _KERNEL_DIR = Path(__file__).resolve().parent _ON = {"1", "on", "true", "yes"} _OFF = {"", "0", "off", "false", "no"} -_X_INDEX_REWRITES = ( - ("uint x_base = cw * 8u;", "uint x_base = route * (codewords * 8u) + cw * 8u;"), - ("uint x_idx = cw * 8u + dim;", "uint x_idx = route * (codewords * 8u) + cw * 8u + dim;"), -) def e8p_m1_enabled() -> bool: @@ -70,38 +64,52 @@ def should_use_e8p_m1( ) -def per_route_x_source() -> str: - """``gather_vqmm_m1.metal`` with the activation row chosen by route.""" - - source = (_KERNEL_DIR / "gather_vqmm_m1.metal").read_text() - for old, new in _X_INDEX_REWRITES: - count = source.count(old) - if count != 1: - raise RuntimeError( - f"gather_vqmm_m1.metal changed: expected one {old!r}, found {count}" - ) - source = source.replace(old, new) - return source - - @cache -def _per_route_kernel(): +def _kernel(): import mlx.core as mx return mx.fast.metal_kernel( - name="ramp_e8p_m1_per_route_x", + name="ramp_e8p_m1", input_names=["x", "codes", "scales", "codebook", "rhs_indices"], output_names=["out"], - source=per_route_x_source(), - header=(_KERNEL_DIR / "common_e8.metal").read_text(), + source=(_KERNEL_DIR / "e8p_m1.metal").read_text(), ) -def _launch_shape(output_dims: int, rows_per_threadgroup: int) -> tuple[int, int]: - lanes_per_row = 256 // rows_per_threadgroup - if rows_per_threadgroup * lanes_per_row != 256 or lanes_per_row < 1: - raise ValueError("rows_per_threadgroup must divide 256") - return lanes_per_row, (output_dims + rows_per_threadgroup - 1) // rows_per_threadgroup +def _chunk(codewords: int, groups: int) -> int: + """Codewords per wide load: a chunk must not straddle a row or scale group.""" + + for chunk in (8, 4): + if codewords % groups == 0 and (codewords // groups) % chunk == 0: + return chunk + raise ValueError(f"E8P M=1 needs codewords per scale group divisible by 4, got {codewords}/{groups}") + + +def _launch(x, codes, scales, codebook, rhs_indices, *, output_dims, rows_per_simdgroup, simdgroups, per_route): + import mlx.core as mx + + from ramp.kernels.gather_vqmm import _qmv_output_dtype + + if rows_per_simdgroup < 1 or simdgroups < 1 or simdgroups > 32: + raise ValueError("rows_per_simdgroup must be >= 1 and simdgroups in 1..32") + routes = int(rhs_indices.size) + rows_per_tile = rows_per_simdgroup * simdgroups + output_dtype = _qmv_output_dtype(x.dtype) + (out,) = _kernel()( + inputs=[mx.contiguous(x), codes, scales, codebook, rhs_indices.reshape(1, routes)], + template=[ + ("ROWS_PER_SG", rows_per_simdgroup), + ("SIMDGROUPS", simdgroups), + ("CHUNK", _chunk(int(codes.shape[2]), int(scales.shape[2]))), + ("PER_ROUTE", int(per_route)), + ("OUT_T", output_dtype), + ], + grid=(32 * simdgroups, (output_dims + rows_per_tile - 1) // rows_per_tile, routes), + threadgroup=(32 * simdgroups, 1, 1), + output_shapes=[(1, routes, output_dims)], + output_dtypes=[output_dtype], + ) + return out def e8p_m1_token_layout( @@ -112,26 +120,19 @@ def e8p_m1_token_layout( rhs_indices, *, output_dims: int, - rows_per_threadgroup: int = 32, - use_threadgroup_codebook: bool = True, + rows_per_simdgroup: int = 4, + simdgroups: int = 1, ): """One activation row against ``top_k`` experts: ``[1, top_k, out]``.""" - from ramp.kernels.gather_vqmm import gather_vqmm_m1_kernel_unchecked - if x.shape[0] != 1 or rhs_indices.ndim != 2 or rhs_indices.shape[0] != 1: raise ValueError("token-layout E8P M=1 needs x [1, in] and rhs_indices [1, top_k]") - return gather_vqmm_m1_kernel_unchecked( - x, - codes, - scales, - codebook, - rhs_indices, + return _launch( + x, codes, scales, codebook, rhs_indices, output_dims=output_dims, - code_bits=16, - rows_per_threadgroup=rows_per_threadgroup, - use_threadgroup_codebook=use_threadgroup_codebook, - use_decoded_codebook=False, + rows_per_simdgroup=rows_per_simdgroup, + simdgroups=simdgroups, + per_route=False, ) @@ -143,33 +144,19 @@ def e8p_m1_per_route( rhs_indices, *, output_dims: int, - rows_per_threadgroup: int = 32, - use_threadgroup_codebook: bool = True, + rows_per_simdgroup: int = 4, + simdgroups: int = 1, ): """Route ``r``'s activation row against expert ``rhs_indices[r]``: ``[routes, out]``.""" - import mlx.core as mx - - from ramp.kernels.gather_vqmm import _qmv_output_dtype - routes = int(rhs_indices.size) if x.ndim != 2 or x.shape[0] != routes: raise ValueError(f"per-route E8P M=1 needs x [{routes}, in], got {tuple(x.shape)}") - lanes_per_row, row_tiles = _launch_shape(output_dims, rows_per_threadgroup) - output_dtype = _qmv_output_dtype(x.dtype) - (out,) = _per_route_kernel()( - inputs=[mx.contiguous(x), codes, scales, codebook, rhs_indices.reshape(1, routes)], - template=[ - ("ROWS_PER_TG", rows_per_threadgroup), - ("LANES_PER_ROW", lanes_per_row), - ("OUT_T", output_dtype), - ("CODE_BITS", 16), - ("USE_THREADGROUP_CODEBOOK", int(use_threadgroup_codebook)), - ("USE_DECODED_CODEBOOK", 0), - ], - grid=(256, row_tiles, routes), - threadgroup=(256, 1, 1), - output_shapes=[(1, routes, output_dims)], - output_dtypes=[output_dtype], + out = _launch( + x, codes, scales, codebook, rhs_indices, + output_dims=output_dims, + rows_per_simdgroup=rows_per_simdgroup, + simdgroups=simdgroups, + per_route=True, ) return out.reshape(routes, output_dims) diff --git a/tests/test_e8p_m1.py b/tests/test_e8p_m1.py index b5a6622..5ff7166 100644 --- a/tests/test_e8p_m1.py +++ b/tests/test_e8p_m1.py @@ -2,7 +2,6 @@ from __future__ import annotations -from pathlib import Path import numpy as np import pytest @@ -44,21 +43,10 @@ def test_flag_is_off_by_default_and_validated(monkeypatch): e8p_m1.e8p_m1_enabled() -def test_per_route_source_only_offsets_the_activation_reads(): - original = (Path(e8p_m1.__file__).parent / "gather_vqmm_m1.metal").read_text() - rewritten = e8p_m1.per_route_x_source() - assert rewritten.count("route * (codewords * 8u)") == 2 - # Nothing else changed: undoing the two rewrites restores the file exactly. - restored = rewritten - for old, new in e8p_m1._X_INDEX_REWRITES: - restored = restored.replace(new, old) - assert restored == original - - -def _layer(code_bits=16, experts=4, out=32, inp=64, seed=0): +def _layer(code_bits=16, experts=4, out=32, inp=64, seed=0, group_size=32): rng = np.random.default_rng(seed) weight = mx.array(rng.normal(size=(experts, out, inp)).astype(np.float32)) - return QuantizedVQSwitchLinear.from_weights(weight, group_size=32, code_bits=code_bits) + return QuantizedVQSwitchLinear.from_weights(weight, group_size=group_size, code_bits=code_bits) def test_switch_linear_routes_m1_16bit_calls_to_the_new_kernels(monkeypatch): @@ -110,30 +98,38 @@ def _dense(layer) -> np.ndarray: @pytest.mark.skipif(not _metal_available(), reason="needs a Metal GPU") -@pytest.mark.parametrize("rows_per_threadgroup", [8, 32]) -def test_token_and_per_route_kernels_match_the_dense_decode(rows_per_threadgroup): - layer = _layer(experts=6, out=96, inp=128, seed=3) +# group 32 -> 4 codewords per group (4-codeword loads); 64 -> 8 (8-codeword +# loads). inp=768 makes lanes take more than one step; out=93 leaves a +# ragged last tile. +@pytest.mark.parametrize("group_size,inp,out", [(32, 128, 96), (64, 768, 93)]) +@pytest.mark.parametrize("rows_per_simdgroup,simdgroups", [(1, 1), (8, 1), (4, 4)]) +def test_token_and_per_route_kernels_match_the_dense_decode(group_size, inp, out, rows_per_simdgroup, simdgroups): + layer = _layer(experts=6, out=out, inp=inp, seed=3, group_size=group_size) + kw = {"output_dims": out, "rows_per_simdgroup": rows_per_simdgroup, "simdgroups": simdgroups} dense = _dense(layer) rng = np.random.default_rng(4) rhs = mx.array([[5, 0, 3]], dtype=mx.uint32) - x = rng.normal(size=(1, 128)).astype(np.float32) - got = e8p_m1.e8p_m1_token_layout( - mx.array(x), layer.codes, layer.effective_scales(), layer.codebook, rhs, - output_dims=96, rows_per_threadgroup=rows_per_threadgroup, - ) + x = rng.normal(size=(1, inp)).astype(np.float32) + got = e8p_m1.e8p_m1_token_layout(mx.array(x), layer.codes, layer.effective_scales(), layer.codebook, rhs, **kw) want = np.stack([x[0] @ dense[e].T for e in (5, 0, 3)])[None] np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) - xr = rng.normal(size=(3, 128)).astype(np.float32) + xr = rng.normal(size=(3, inp)).astype(np.float32) got = e8p_m1.e8p_m1_per_route( - mx.array(xr), layer.codes, layer.effective_scales(), layer.codebook, rhs.reshape(-1), - output_dims=96, rows_per_threadgroup=rows_per_threadgroup, + mx.array(xr), layer.codes, layer.effective_scales(), layer.codebook, rhs.reshape(-1), **kw ) want = np.stack([xr[r] @ dense[e].T for r, e in enumerate((5, 0, 3))]) np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) +def test_chunk_never_straddles_a_scale_group(): + assert e8p_m1._chunk(512, 8) == 8 # V4-Flash gate/up: 64 codewords per group + assert e8p_m1._chunk(16, 4) == 4 + with pytest.raises(ValueError, match="divisible by 4"): + e8p_m1._chunk(12, 2) + + def test_benchmark_synthetic_layers_match_the_shipped_byte_accounting(): from benchmarks import bench_e8p_m1 as bench From ec08ddc48409f630f70ef4e0866de2f03a8a1c0d Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:18:30 -0800 Subject: [PATCH 15/74] ramp: make the tuned E8P M=1 kernel the default on GPUs without NAX With RAMP_E8P_M1 unset (or auto), 16-bit E8P codes at M=1 now use e8p_m1.metal on any Metal GPU where ramp.kernels.nax.is_available() is false, which means M1-M4. NAX GPUs and CPU-only runs are unchanged. RAMP_E8P_M1=0 turns it off and =1 forces it on. Across the 29 test files that touch these modules on the M4 Max, the only difference is one flaky speculative-runtime test, which failed only with the default off. The kill test passed by 1-3% on a loaded machine, so it should be rerun on an idle one. --- .../2026-11-05-phase0-kill-tests.md | 6 ++-- .../2026-11-05-v41-plan-and-phase0-status.md | 6 ++-- src/ramp/kernels/e8p_m1.py | 22 ++++++++++++--- src/ramp/nn/switch_linear.py | 4 +-- tests/test_e8p_m1.py | 28 ++++++++++++++++--- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index d73b9d4..376d84b 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -80,8 +80,10 @@ script above runs on the Mac. lanes load 8 codewords at once, and the abs grid is decoded once per threadgroup into half2 pairs, with signs applied by XOR. A loads-only stub of the same kernel reads at ~410 GB/s, so decode ALU is still half the - time. Not yet the default: it clears the bar by 1-3% while other jobs were - loading the machine. + time. Since 2026-11-05 it is the default on any Metal GPU without NAX + (`RAMP_E8P_M1` unset or `auto`); `RAMP_E8P_M1=0` turns it off, `=1` forces + it on. It clears the bar by only 1-3%, measured while other jobs were + loading the machine, so rerun the kill test on an idle machine. - `bench_dsv4_mxfp8_residents.py` v1 also synced after every call. v2 (`dsv4_mxfp8_resident_kill_test_v2`) chains 43 dependent calls per `mx.eval`. On the M4 Max the verdict flips from KILL to **PASS** in two runs: diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index ec9991c..7b2bd70 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -108,8 +108,10 @@ or Mac-local files), plus 22 newly passing tests. lanes load 8 codewords at once, and the abs grid is decoded once per threadgroup into half2 pairs, with signs applied by XOR. A loads-only stub of the same kernel reads at ~410 GB/s, so decode ALU is still half the - time. Not yet the default: it clears the bar by 1-3% while other jobs were - loading the machine. + time. Since 2026-11-05 it is the default on any Metal GPU without NAX + (`RAMP_E8P_M1` unset or `auto`); `RAMP_E8P_M1=0` turns it off, `=1` forces + it on. It clears the bar by only 1-3%, measured while other jobs were + loading the machine, so rerun the kill test on an idle machine. - `bench_dsv4_mxfp8_residents.py` v1 also synced after every call. v2 (`dsv4_mxfp8_resident_kill_test_v2`) chains 43 dependent calls per `mx.eval`. On the M4 Max the verdict flips from KILL to **PASS** in two runs: diff --git a/src/ramp/kernels/e8p_m1.py b/src/ramp/kernels/e8p_m1.py index c54c29b..30e4031 100644 --- a/src/ramp/kernels/e8p_m1.py +++ b/src/ramp/kernels/e8p_m1.py @@ -7,8 +7,9 @@ threadgroup per output element. ``gather_vqmm_m1.metal`` already carries an E8P branch; nothing dispatched to it. -With ``RAMP_E8P_M1=1`` (opt-in until ``benchmarks/bench_e8p_m1.py`` passes on -the target Mac), M=1 calls with 16-bit codes use ``e8p_m1.metal``: +On a Metal GPU without the NAX kernels (M1-M4), M=1 calls with 16-bit codes +use ``e8p_m1.metal`` by default, since ``benchmarks/bench_e8p_m1.py`` passed +on the M4 Max (2026-11-05). ``RAMP_E8P_M1=1`` forces it on, ``=0`` off: - :func:`e8p_m1_token_layout` for gate/up (one activation row, top-k experts). @@ -38,7 +39,18 @@ E8P_M1_ENV = "RAMP_E8P_M1" _KERNEL_DIR = Path(__file__).resolve().parent _ON = {"1", "on", "true", "yes"} -_OFF = {"", "0", "off", "false", "no"} +_OFF = {"0", "off", "false", "no"} + + +@cache +def default_enabled() -> bool: + """On for a Metal GPU without NAX; NAX GPUs keep their tensor-unit kernels.""" + + import mlx.core as mx + + from ramp.kernels import nax + + return bool(mx.metal.is_available()) and not nax.is_available() def e8p_m1_enabled() -> bool: @@ -47,7 +59,9 @@ def e8p_m1_enabled() -> bool: return True if value in _OFF: return False - raise ValueError(f"{E8P_M1_ENV} must be on/off (1/0), got {value!r}") + if value in ("", "auto"): + return default_enabled() + raise ValueError(f"{E8P_M1_ENV} must be on/off/auto (1/0), got {value!r}") def should_use_e8p_m1( diff --git a/src/ramp/nn/switch_linear.py b/src/ramp/nn/switch_linear.py index 172407f..c42e8a3 100644 --- a/src/ramp/nn/switch_linear.py +++ b/src/ramp/nn/switch_linear.py @@ -339,8 +339,8 @@ def __call__(self, x: mx.array, indices: mx.array, *, sorted_indices: bool = Fal sorted_indices=sorted_indices, route_strategy=self.route_strategy, ): - # Opt-in (RAMP_E8P_M1=1): 16-bit codes at M=1 on GPUs without - # the NAX tensor-unit kernels, instead of the scalar fallback. + # 16-bit codes at M=1 on GPUs without the NAX tensor-unit + # kernels (default there; RAMP_E8P_M1=0/1 overrides). y = e8p_m1_token_layout( flat_x, self.codes, diff --git a/tests/test_e8p_m1.py b/tests/test_e8p_m1.py index 5ff7166..a6cdc73 100644 --- a/tests/test_e8p_m1.py +++ b/tests/test_e8p_m1.py @@ -35,9 +35,29 @@ def test_decision_table_with_the_flag_on(monkeypatch, kwargs, expected): assert e8p_m1.should_use_e8p_m1(**args) is expected -def test_flag_is_off_by_default_and_validated(monkeypatch): - monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) +@pytest.mark.parametrize("metal,nax,expected", [(True, False, True), (True, True, False), (False, False, False)]) +def test_default_is_on_only_for_a_metal_gpu_without_nax(monkeypatch, metal, nax, expected): + from ramp.kernels import nax as nax_module + + monkeypatch.setattr(mx.metal, "is_available", lambda: metal) + monkeypatch.setattr(nax_module, "is_available", lambda: nax) + e8p_m1.default_enabled.cache_clear() + try: + for value in (None, "", "auto"): + if value is None: + monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) + else: + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, value) + assert e8p_m1.e8p_m1_enabled() is expected + finally: + e8p_m1.default_enabled.cache_clear() + + +def test_flag_overrides_the_default_and_is_validated(monkeypatch): + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "0") assert e8p_m1.should_use_e8p_m1(code_bits=16, tokens=1, sorted_indices=False, route_strategy="auto") is False + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "1") + assert e8p_m1.should_use_e8p_m1(code_bits=16, tokens=1, sorted_indices=False, route_strategy="auto") is True monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "maybe") with pytest.raises(ValueError, match=e8p_m1.E8P_M1_ENV): e8p_m1.e8p_m1_enabled() @@ -74,14 +94,14 @@ def fake_route(x, codes, scales, codebook, rhs, *, output_dims): def test_switch_linear_leaves_8bit_and_flag_off_calls_alone(monkeypatch): def boom(*args, **kwargs): # pragma: no cover - must not be called - raise AssertionError("opt-in kernel used without the flag or for 8-bit codes") + raise AssertionError("E8P M=1 kernel used with the flag off or for 8-bit codes") monkeypatch.setattr(switch_linear, "e8p_m1_token_layout", boom) monkeypatch.setattr(switch_linear, "e8p_m1_per_route", boom) monkeypatch.setattr(switch_linear, "gather_vqmm", lambda *a, **k: mx.zeros((3, 32))) monkeypatch.setattr(switch_linear, "gather_vqmm_m1_kernel", lambda *a, **k: mx.zeros((1, 3, 32))) indices = mx.array([[0, 2, 3]], dtype=mx.uint32) - monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "0") _layer(code_bits=16)(mx.zeros((1, 64)), indices) monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "1") _layer(code_bits=8)(mx.zeros((1, 64)), indices) From 705ed3475ae15ca376c6c9ad8fc0c41cf691f808 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:42:16 -0800 Subject: [PATCH 16/74] ramp: serve MTP verify windows (<= 32 tokens) with the E8P kernel on non-NAX GPUs The tuned E8P kernel now takes any number of routes: route r multiplies activation row r / X_DIV (top_k for gate/up, 1 for down) by expert rhs_indices[r]. should_use_e8p_m1 accepts 1-32 token rows, and the GLU's shared sorted-route prefill path declines those calls, so verify windows no longer fall to the scalar or sorted-route kernels on M1-M4. Synthetic V4-Flash FFN on the M4 Max (256 experts, top-6, 43 chained FFNs per sync): 3.9x the previous path at 1 token, 4.7-5.5x at 2-8, 2.6-3.3x at 11-21 and 9x at 32, outputs within 5.7e-4. Prefill above 32 tokens is unchanged. default_enabled() is no longer cached (nax.is_available already is), and the no-native fallback test now pins RAMP_E8P_M1=0. --- .../e8p-verify/verify_bench.py | 42 ++++++++++++++ .../e8p-verify/verify_bench_m4max.txt | 8 +++ .../2026-11-05-phase0-kill-tests.md | 6 ++ .../2026-11-05-v41-plan-and-phase0-status.md | 6 ++ src/ramp/kernels/e8p_m1.metal | 13 +++-- src/ramp/kernels/e8p_m1.py | 31 ++++++---- src/ramp/models/glm4_moe_adapter.py | 8 +++ src/ramp/nn/switch_linear.py | 2 +- tests/test_e8p_m1.py | 56 +++++++++++++++---- tests/test_switch_routing.py | 3 + 10 files changed, 145 insertions(+), 30 deletions(-) create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench.py create mode 100644 artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench_m4max.txt diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench.py b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench.py new file mode 100644 index 0000000..8edddf2 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench.py @@ -0,0 +1,42 @@ +import os, sys, time, statistics, warnings +warnings.filterwarnings("ignore") +import numpy as np, mlx.core as mx +sys.path.insert(0, ".") +from benchmarks.bench_e8p_m1 import synthetic_projection +from ramp.kernels import e8p_m1 +from ramp.models.glm4_moe_adapter import QuantizedVQSwitchGLU +E = 256 +glu = QuantizedVQSwitchGLU( + gate_proj=synthetic_projection(experts=E, out=2048, inp=4096, group_size=512, seed=1), + up_proj=synthetic_projection(experts=E, out=2048, inp=4096, group_size=512, seed=2), + down_proj=synthetic_projection(experts=E, out=4096, inp=2048, group_size=512, seed=3), +) +per_expert = sum((p.codes.nbytes + p.scales.nbytes) // E for p in (glu.gate_proj, glu.up_proj, glu.down_proj)) +rng = np.random.default_rng(0) +def routes(m): + return [mx.array(np.stack([np.sort(rng.choice(E, 6, replace=False)) for _ in range(m)])[None].astype(np.uint32)) for _ in range(32)] +def chain(x, rs): + for r in rs: + y = glu(x, r) # [1, m, 6, hidden] + x = (x + y.sum(axis=2) * 1e-3).astype(x.dtype) + return x +def ms_per_ffn(m, rs, n=43, samples=12): + x = mx.array(rng.normal(size=(1, m, 4096)).astype(np.float16)) + def one(i): + seq = [rs[(i * n + j) % len(rs)] for j in range(n)] + t = time.perf_counter(); mx.eval(chain(x, seq)); return (time.perf_counter() - t) * 1e3 / n + one(0); one(1) + return statistics.median(one(i) for i in range(samples)) +for m in (1, 2, 4, 8, 11, 16, 21, 32): + rs = routes(m) + distinct = statistics.mean(len(set(np.array(r).reshape(-1).tolist())) for r in rs) + out = {} + for flag in ("0", "1"): + os.environ[e8p_m1.E8P_M1_ENV] = flag + out[flag] = ms_per_ffn(m, rs) + x = mx.array(rng.normal(size=(1, m, 4096)).astype(np.float16)) + os.environ[e8p_m1.E8P_M1_ENV] = "0"; a = np.array(glu(x, rs[0]).astype(mx.float32)) + os.environ[e8p_m1.E8P_M1_ENV] = "1"; b = np.array(glu(x, rs[0]).astype(mx.float32)) + diff = np.max(np.abs(a - b)) / np.max(np.abs(a)) + gbs = distinct * per_expert / (out["1"] / 1e3) / 1e9 + print(f"M={m:2d} distinct experts {distinct:5.1f} old {out['0']:.3f} ms new {out['1']:.3f} ms x{out['0']/out['1']:.2f} {gbs:.0f} GB/s distinct diff {diff:.1e}") diff --git a/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench_m4max.txt b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench_m4max.txt new file mode 100644 index 0000000..4b6a3d2 --- /dev/null +++ b/artifacts/benchmarks/phase0-MAXBOOK-20261105-192445/e8p-verify/verify_bench_m4max.txt @@ -0,0 +1,8 @@ +M= 1 distinct experts 6.0 old 0.770 ms new 0.195 ms x3.96 197 GB/s distinct diff 3.0e-04 +M= 2 distinct experts 11.8 old 1.445 ms new 0.314 ms x4.60 240 GB/s distinct diff 5.7e-04 +M= 4 distinct experts 23.1 old 2.809 ms new 0.537 ms x5.23 275 GB/s distinct diff 2.8e-04 +M= 8 distinct experts 43.8 old 5.532 ms new 0.998 ms x5.54 280 GB/s distinct diff 4.7e-04 +M=11 distinct experts 59.0 old 4.444 ms new 1.360 ms x3.27 277 GB/s distinct diff 4.5e-04 +M=16 distinct experts 79.7 old 5.366 ms new 1.932 ms x2.78 263 GB/s distinct diff 5.0e-04 +M=21 distinct experts 99.4 old 7.073 ms new 2.539 ms x2.79 250 GB/s distinct diff 5.1e-04 +M=32 distinct experts 136.2 old 39.129 ms new 4.493 ms x8.71 194 GB/s distinct diff 5.2e-04 diff --git a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md index 376d84b..4f3f2d8 100644 --- a/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md +++ b/docs/deepseek-v4-flash/2026-11-05-phase0-kill-tests.md @@ -66,6 +66,12 @@ script above runs on the Mac. - `benchmarks/bench_dsv4_mtp_headline.py` validates speculative rows against `EXPECTED_VERIFY_IMPLEMENTATION = "nax_e8p_m32n64"`, so MTP headline rows cannot validate on an M4 until the verify window has a non-NAX E8P kernel. + On a synthetic V4-Flash FFN (256 experts, top-6, 43 chained FFNs per sync) + the same kernel now also serves MTP verify windows (<= 32 tokens): 2.6-5.5x + the previous M4 path at 2-21 tokens and 9x at 32, outputs within 5.7e-4. + It still is not traced as a verify implementation, so the MTP headline + harness, which expects `nax_e8p_m32n64`, needs updating before M4 rows can + validate. - Decoding each E8P codeword once was tried on the M4 Max on 2026-11-05 and made the kernel slower (113 vs 154 GB/s, kernel alone, 256 calls per sync). The compiler already lifts the per-codeword work out of the 8-dimension loop. diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md index 7b2bd70..0df2a31 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-plan-and-phase0-status.md @@ -94,6 +94,12 @@ or Mac-local files), plus 22 newly passing tests. - MTP headline rows require the M5-only `nax_e8p_m32n64`, so they won't validate on an M4. + On a synthetic V4-Flash FFN (256 experts, top-6, 43 chained FFNs per sync) + the same kernel now also serves MTP verify windows (<= 32 tokens): 2.6-5.5x + the previous M4 path at 2-21 tokens and 9x at 32, outputs within 5.7e-4. + It still is not traced as a verify implementation, so the MTP headline + harness, which expects `nax_e8p_m32n64`, needs updating before M4 rows can + validate. - Decoding each E8P codeword once was tried on the M4 Max on 2026-11-05 and made the kernel slower (113 vs 154 GB/s, kernel alone, 256 calls per sync). The compiler already lifts the per-codeword work out of the 8-dimension loop. diff --git a/src/ramp/kernels/e8p_m1.metal b/src/ramp/kernels/e8p_m1.metal index 96b55d5..e3c2797 100644 --- a/src/ramp/kernels/e8p_m1.metal +++ b/src/ramp/kernels/e8p_m1.metal @@ -1,4 +1,6 @@ -// Single-token (M=1) gather matvec for 16-bit E8P codes on GPUs without NAX. +// Few-token (M=1 decode, M<=32 verify) gather matvec for 16-bit E8P codes on +// GPUs without NAX. One threadgroup column per route: route r multiplies +// activation row r / X_DIV by expert rhs_indices[r]. // // Each simdgroup owns ROWS_PER_SG output rows. A lane takes CHUNK (4 or 8) // consecutive codewords: one 8- or 16-byte load per row, so a simdgroup reads @@ -15,21 +17,20 @@ // each threadgroup first decodes the abs grid into half2 pairs, and a // codeword then costs one threadgroup read plus one XOR per pair for signs. // -// PER_ROUTE=0: one activation row for every route (gate/up). -// PER_ROUTE=1: route r reads activation row r (down). +// X_DIV=top_k: token t's routes share activation row t (gate/up). +// X_DIV=1: every route has its own activation row (down). uint simd_lane = thread_index_in_simdgroup; uint simd_id = simdgroup_index_in_threadgroup; uint tg_lane = thread_position_in_threadgroup.x; uint row_tile = threadgroup_position_in_grid.y; uint route = threadgroup_position_in_grid.z; -uint top_k = uint(rhs_indices_shape[1]); -uint expert = uint(rhs_indices[route % top_k]); +uint expert = uint(rhs_indices[route]); uint out_dim = uint(codes_shape[1]); uint codewords = uint(codes_shape[2]); uint groups = uint(scales_shape[2]); uint codewords_per_group = codewords / groups; -uint x_row = uint(PER_ROUTE) != 0u ? route * codewords * 8u : 0u; +uint x_row = (route / uint(X_DIV)) * codewords * 8u; // (1024 + n) * 0.5 - 516 = (n - 8) / 2, exact in half. threadgroup uint4 decoded[256]; diff --git a/src/ramp/kernels/e8p_m1.py b/src/ramp/kernels/e8p_m1.py index 30e4031..43ca060 100644 --- a/src/ramp/kernels/e8p_m1.py +++ b/src/ramp/kernels/e8p_m1.py @@ -37,14 +37,19 @@ ] E8P_M1_ENV = "RAMP_E8P_M1" +#: Token rows the kernel serves: M=1 decode and MTP verify windows. Wider +#: calls (prefill) keep the sorted-route kernels, which share expert decode. +E8P_MAX_TOKENS = 32 _KERNEL_DIR = Path(__file__).resolve().parent _ON = {"1", "on", "true", "yes"} _OFF = {"0", "off", "false", "no"} -@cache def default_enabled() -> bool: - """On for a Metal GPU without NAX; NAX GPUs keep their tensor-unit kernels.""" + """On for a Metal GPU without NAX; NAX GPUs keep their tensor-unit kernels. + + Not cached: ``nax.is_available`` already is, and tests patch it. + """ import mlx.core as mx @@ -71,7 +76,7 @@ def should_use_e8p_m1( return ( code_bits == 16 - and tokens == 1 + and 1 <= tokens <= E8P_MAX_TOKENS and not sorted_indices and route_strategy in ("auto", "direct") and e8p_m1_enabled() @@ -99,7 +104,7 @@ def _chunk(codewords: int, groups: int) -> int: raise ValueError(f"E8P M=1 needs codewords per scale group divisible by 4, got {codewords}/{groups}") -def _launch(x, codes, scales, codebook, rhs_indices, *, output_dims, rows_per_simdgroup, simdgroups, per_route): +def _launch(x, codes, scales, codebook, rhs_indices, *, output_dims, rows_per_simdgroup, simdgroups, x_div): import mlx.core as mx from ramp.kernels.gather_vqmm import _qmv_output_dtype @@ -110,12 +115,12 @@ def _launch(x, codes, scales, codebook, rhs_indices, *, output_dims, rows_per_si rows_per_tile = rows_per_simdgroup * simdgroups output_dtype = _qmv_output_dtype(x.dtype) (out,) = _kernel()( - inputs=[mx.contiguous(x), codes, scales, codebook, rhs_indices.reshape(1, routes)], + inputs=[mx.contiguous(x), codes, scales, codebook, rhs_indices.reshape(-1)], template=[ ("ROWS_PER_SG", rows_per_simdgroup), ("SIMDGROUPS", simdgroups), ("CHUNK", _chunk(int(codes.shape[2]), int(scales.shape[2]))), - ("PER_ROUTE", int(per_route)), + ("X_DIV", int(x_div)), ("OUT_T", output_dtype), ], grid=(32 * simdgroups, (output_dims + rows_per_tile - 1) // rows_per_tile, routes), @@ -137,17 +142,19 @@ def e8p_m1_token_layout( rows_per_simdgroup: int = 4, simdgroups: int = 1, ): - """One activation row against ``top_k`` experts: ``[1, top_k, out]``.""" + """Token ``t``'s row against its ``top_k`` experts: ``[tokens, top_k, out]``.""" - if x.shape[0] != 1 or rhs_indices.ndim != 2 or rhs_indices.shape[0] != 1: - raise ValueError("token-layout E8P M=1 needs x [1, in] and rhs_indices [1, top_k]") - return _launch( + if x.ndim != 2 or rhs_indices.ndim != 2 or rhs_indices.shape[0] != x.shape[0]: + raise ValueError("token-layout E8P needs x [tokens, in] and rhs_indices [tokens, top_k]") + tokens, top_k = int(rhs_indices.shape[0]), int(rhs_indices.shape[1]) + out = _launch( x, codes, scales, codebook, rhs_indices, output_dims=output_dims, rows_per_simdgroup=rows_per_simdgroup, simdgroups=simdgroups, - per_route=False, + x_div=top_k, ) + return out.reshape(tokens, top_k, output_dims) def e8p_m1_per_route( @@ -171,6 +178,6 @@ def e8p_m1_per_route( output_dims=output_dims, rows_per_simdgroup=rows_per_simdgroup, simdgroups=simdgroups, - per_route=True, + x_div=1, ) return out.reshape(routes, output_dims) diff --git a/src/ramp/models/glm4_moe_adapter.py b/src/ramp/models/glm4_moe_adapter.py index 2afdd13..510f3a5 100644 --- a/src/ramp/models/glm4_moe_adapter.py +++ b/src/ramp/models/glm4_moe_adapter.py @@ -9,6 +9,7 @@ import mlx.nn as nn from ramp.kernels import nax +from ramp.kernels.e8p_m1 import should_use_e8p_m1 from ramp.nn.switch_linear import HighPrecisionSwitchLinear, QuantizedVQSwitchLinear from ramp.ops.vq_switch import ( _expert_block_tile_descriptors, @@ -157,6 +158,13 @@ def num_experts(self) -> int: return self.gate_proj.num_experts def _can_use_shared_sorted_route_prefill(self, x: mx.array, indices: mx.array) -> bool: + if indices.ndim >= 2 and should_use_e8p_m1( + code_bits=self.gate_proj.code_bits, + tokens=indices.size // indices.shape[-1], + sorted_indices=False, + route_strategy=self.gate_proj.route_strategy, + ): + return False # few-token 16-bit E8P: the per-projection kernel is faster return ( indices.size >= 64 and x.ndim >= 2 diff --git a/src/ramp/nn/switch_linear.py b/src/ramp/nn/switch_linear.py index c42e8a3..0b98441 100644 --- a/src/ramp/nn/switch_linear.py +++ b/src/ramp/nn/switch_linear.py @@ -393,7 +393,7 @@ def __call__(self, x: mx.array, indices: mx.array, *, sorted_indices: bool = Fal flat_indices = indices.reshape((-1,)) if indices.ndim >= 2 and should_use_e8p_m1( code_bits=self.code_bits, - tokens=route_shape[0], + tokens=flat_indices.shape[0] // route_shape[-1], sorted_indices=sorted_indices, route_strategy=self.route_strategy, ): diff --git a/tests/test_e8p_m1.py b/tests/test_e8p_m1.py index a6cdc73..cef5f38 100644 --- a/tests/test_e8p_m1.py +++ b/tests/test_e8p_m1.py @@ -23,7 +23,10 @@ def _metal_available() -> bool: [ ({}, True), ({"code_bits": 8}, False), - ({"tokens": 2}, False), + ({"tokens": 2}, True), # MTP verify window + ({"tokens": 32}, True), + ({"tokens": 33}, False), # prefill keeps the sorted-route kernels + ({"tokens": 0}, False), ({"sorted_indices": True}, False), ({"route_strategy": "sorted_tiled"}, False), ({"route_strategy": "direct"}, True), @@ -41,16 +44,12 @@ def test_default_is_on_only_for_a_metal_gpu_without_nax(monkeypatch, metal, nax, monkeypatch.setattr(mx.metal, "is_available", lambda: metal) monkeypatch.setattr(nax_module, "is_available", lambda: nax) - e8p_m1.default_enabled.cache_clear() - try: - for value in (None, "", "auto"): - if value is None: - monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) - else: - monkeypatch.setenv(e8p_m1.E8P_M1_ENV, value) - assert e8p_m1.e8p_m1_enabled() is expected - finally: - e8p_m1.default_enabled.cache_clear() + for value in (None, "", "auto"): + if value is None: + monkeypatch.delenv(e8p_m1.E8P_M1_ENV, raising=False) + else: + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, value) + assert e8p_m1.e8p_m1_enabled() is expected def test_flag_overrides_the_default_and_is_validated(monkeypatch): @@ -143,6 +142,41 @@ def test_token_and_per_route_kernels_match_the_dense_decode(group_size, inp, out np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) +@pytest.mark.skipif(not _metal_available(), reason="needs a Metal GPU") +@pytest.mark.parametrize("tokens", [2, 5]) +def test_multi_token_verify_layouts_match_the_dense_decode(tokens): + layer = _layer(experts=6, out=64, inp=128, seed=5) + dense = _dense(layer) + rng = np.random.default_rng(6) + rhs = np.stack([rng.choice(6, 3, replace=False) for _ in range(tokens)]).astype(np.uint32) + + x = rng.normal(size=(tokens, 128)).astype(np.float32) + got = e8p_m1.e8p_m1_token_layout( + mx.array(x), layer.codes, layer.effective_scales(), layer.codebook, mx.array(rhs), output_dims=64 + ) + want = np.stack([[x[t] @ dense[e].T for e in rhs[t]] for t in range(tokens)]) + np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) + + xr = rng.normal(size=(tokens * 3, 128)).astype(np.float32) + got = e8p_m1.e8p_m1_per_route( + mx.array(xr), layer.codes, layer.effective_scales(), layer.codebook, mx.array(rhs.reshape(-1)), output_dims=64 + ) + want = np.stack([xr[r] @ dense[e].T for r, e in enumerate(rhs.reshape(-1))]) + np.testing.assert_allclose(np.array(got.astype(mx.float32)), want, rtol=1e-4, atol=1e-3) + + +def test_glu_skips_the_sorted_prefill_path_for_few_token_e8p(monkeypatch): + from ramp.models.glm4_moe_adapter import QuantizedVQSwitchGLU + + glu = QuantizedVQSwitchGLU(gate_proj=_layer(out=32, inp=64), up_proj=_layer(out=32, inp=64), down_proj=_layer(out=64, inp=32)) + x = mx.zeros((1, 16, 64)) + indices = mx.zeros((1, 16, 6), dtype=mx.uint32) # 96 routes: past the sorted-prefill threshold + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "1") + assert glu._can_use_shared_sorted_route_prefill(x, indices) is False + monkeypatch.setenv(e8p_m1.E8P_M1_ENV, "0") + assert glu._can_use_shared_sorted_route_prefill(x, indices) is (glu.gate_proj.route_strategy != "direct") + + def test_chunk_never_straddles_a_scale_group(): assert e8p_m1._chunk(512, 8) == 8 # V4-Flash gate/up: 64 codewords per group assert e8p_m1._chunk(16, 4) == 4 diff --git a/tests/test_switch_routing.py b/tests/test_switch_routing.py index 97f6631..0269247 100644 --- a/tests/test_switch_routing.py +++ b/tests/test_switch_routing.py @@ -414,6 +414,9 @@ def recording_helper(x_arg, codes, scales, codebook, sorted_rhs, sorted_lhs, **k monkeypatch.setattr(glm4_moe_adapter.nax, "is_available", lambda: False) monkeypatch.setattr(glm4_moe_adapter, "gather_vqmm_sorted_routes", recording_helper) + # Without NAX, 16-bit E8P at <= 32 tokens now takes ramp.kernels.e8p_m1 by + # default; this pins the sorted-route fallback behind it. + monkeypatch.setenv("RAMP_E8P_M1", "0") with pytest.warns( RuntimeWarning, From d5ad5c001349e3ad76c93e3888cf92ad768bccec Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:21:05 -0800 Subject: [PATCH 17/74] runpod: V4.1-Flash REAP kit (teacher pass, candidate scoring, byte-exact pruner) Adds scripts/runpod/v41_reap/, a kit that runs on a rented 8xH200 pod and produces new DeepSeek-V4.1-Flash-1017 artifacts, so the Mac downloads a pruned checkpoint instead of the 510 GB source. - collect.py: runs DeepSeek's reference torch code under torchrun (model.py sha256 is pinned) with MoE.forward patched to record per-session REAP stats, imatrix sums (hidden/down), route traces and top-2048 eval logits. It writes KEEP's session schema, so finalize-reap, reap-select and finalize consume the output unchanged. A masked-router student pass scores each candidate expert count against the teacher (KL, NLL change, top-1 agreement) from a single converted checkpoint. - prune.py: copies safetensors bytes (FP8/FP4 are never decoded, no torch), renumbers the kept experts, slices gate weight/bias/bias_vl, leaves mtp.* untouched, updates both configs, and writes the index and SHA256SUMS. - build_pack.py: builds an English-only calibration and held-out pack from public datasets in V4.1's own chat/DSML format, with group-disjoint splits. - score.py and pod.sh (fetch/pack/gpu/prune stages; the GPU stage stops the pod at MAX_GPU_HOURS). - Runbook, plus the calibration-data research (including vetting of the five suggested datasets). All of this is verified on CPU only (tests/test_runpod_v41_reap.py and a local end-to-end through the real finalizers). Nothing has run on a GPU yet. --- .../2026-11-05-v41-runpod-reap-runbook.md | 203 +++++++ .../2026-11-05-v41-reap-calibration-data.md | 327 +++++++++++ scripts/runpod/v41_reap/build_pack.py | 381 ++++++++++++ scripts/runpod/v41_reap/collect.py | 552 ++++++++++++++++++ scripts/runpod/v41_reap/pod.sh | 119 ++++ scripts/runpod/v41_reap/prune.py | 208 +++++++ scripts/runpod/v41_reap/score.py | 51 ++ tests/test_runpod_v41_reap.py | 343 +++++++++++ 8 files changed, 2184 insertions(+) create mode 100644 docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md create mode 100644 docs/research/2026-11-05-v41-reap-calibration-data.md create mode 100644 scripts/runpod/v41_reap/build_pack.py create mode 100644 scripts/runpod/v41_reap/collect.py create mode 100755 scripts/runpod/v41_reap/pod.sh create mode 100644 scripts/runpod/v41_reap/prune.py create mode 100644 scripts/runpod/v41_reap/score.py create mode 100644 tests/test_runpod_v41_reap.py diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md new file mode 100644 index 0000000..1ac3e76 --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md @@ -0,0 +1,203 @@ +# DeepSeek-V4.1-Flash REAP on RunPod: runbook + +Date: 2026-11-05. Budget: $497. Kit: `scripts/runpod/v41_reap/`. + +## Goal + +The V4-Flash artifacts are gone. This run makes new V4.1 artifacts on rented +GPUs so that the Mac downloads a smaller, pruned checkpoint instead of the +510 GB source. + +| Output | Where (on the volume) | Size (est.) | Used for | +|---|---|---|---| +| Pruned HF checkpoint (FP8 dense, FP4 experts, Engram, drafter untouched) | `/workspace/v41-reapN/` | ~366 GB at N=192, ~342 GB at 160 | V4.1 port, E8P conversion | +| REAP stats, survivor tables, reap-select reports | `/workspace/out/` | < 50 MB | prune choice, hot set | +| Route traces (first 2,048 tokens of each session) | in each session npz | small | residency replay (SSD tier) | +| E8P importance stats (imatrix `hidden` + `down`) | `out/calibration/sessions/*.npz` | ~0.9 GB per session, ~25 sessions | E8P/VQ fitting (`finalize`) | +| Teacher top-2048 logits for the eval split | `out/eval/sessions/*.npz` | ~5 GB | KL gates, MTP and distillation | +| Candidate scores (KL, NLL change, top-1 agreement per domain) | `out/candidate-scores.json` | tiny | choosing N | + +## What is verified, and what is not + +**Verified on CPU on this Mac**, in 7 tests in `tests/test_runpod_v41_reap.py` +plus a manual end-to-end: + +- The patched `MoE.forward` gives bit-identical output to a copy of the + reference Gate/Expert/MoE. +- REAP and imatrix sums match a direct float64 computation. +- KEEP's own validators accept the session files, and `finalize-reap`, + `reap-select` and `finalize` run unchanged on them. +- The head capture matches full logits. +- A masked-router student scored against its own teacher gives KL 0. +- The pruner copies bytes exactly, renumbers experts, slices `weight`, `bias` + and `bias_vl`, and leaves `mtp.*` alone. +- The pack builder keeps eval and calibration groups disjoint and drops + non-English text. +- Real rows from Open-SWE-Traces, Nemotron-Terminal and Toucan render through + V4.1's own `encoding.py` and tokenizer. + +**Not verified until the pod runs:** + +- the reference code under torchrun with 8 GPUs and NCCL; +- throughput (`progress.jsonl` reports tok/s per session); +- GPU memory headroom at 16K tokens; +- that each dataset streams with the field names assumed. They were checked + against one row of each via the HF datasets-server. + +## Calibration mix + +The research is in `docs/research/2026-11-05-v41-reap-calibration-data.md`. In +summary: + +- **Size:** about 3.2M calibration tokens and 0.45M eval tokens. +- **Language:** English only. A record is dropped if it contains any CJK, + Hangul or Kana, or more than 2% of its letters are outside Latin. +- **Agentic and tool-loop share:** 60%. The sources are: + - Open-SWE-Traces (DeepSeek-V4-Flash-0731 OpenHands trajectories; Python 400K, TS 300K, Rust 300K); + - SWE-rebench-V2 PRs as issue-to-patch pairs (400K); + - Nemotron-Terminal (400K); + - Toucan multi-turn MCP traces (450K). +- **The rest:** + - cyber: CIRCL advisories plus security StackExchange, 300K; + - networking: RFC 9000-9800 plus networkengineering, serverfault and unix StackExchange, 250K; + - business: finepdfs business-filtered plus Enron, 250K; + - English: WildChat human prompts plus finewiki, 150K. +- **Held-out split:** one group bucket in ten goes to eval. Groups are + repository, task, RFC, URL host, CVE and tool set. + +Differences from the research mix: + +- **No Primus-FineWeb.** It is gated. +- **No WildChat responses**, only the human prompts. The responses are OpenAI + output. +- **No stackv2 plain-code files.** The shards have no language index, so I + doubled SWE-rebench PRs instead. +- **No 32K-64K long slice.** `--max-seq-len` is 16384. Longer records are + truncated, not windowed. +- **Assistant turns are not regenerated by V4.1.** That would add a generation + stage, so it is a follow-up. + +Jack's five candidates: + +- Open-SWE-Traces is the top source. +- CoderForge is suited to eval only and is not used yet. +- Vyber07/cyber-security, Fable-5.1 and Axion are rejected. Fable-5.1 is + Claude-distilled with placeholder tool arguments. + +## Cost and time (estimates) + +The prices come from runpod.io/pricing on 2026-11-05. Secure H200 is $5.29/GPU/h. + +| Stage | Pod | Time | Cost | +|---|---|---|---| +| Network volume, 1.5 TB, ~5 days | — | — | ~$15 | +| `fetch`: 510 GB download plus sha256 of every file | CPU | 1-3 h | < $2 | +| `pack` | CPU | 1-2 h | < $2 | +| `gpu`: install, convert (~1 h), teacher pass (~3.7M tokens), 4 student passes (4 × 0.45M) | 8×H200 | 2.5-3.5 h | $105-150 | +| `prune N`, then the Mac pulls ~366 GB | CPU | 1-4 h | < $2 | +| **Expected total** | | | **~$125-170** | +| **Worst case: the watchdog fires at `MAX_GPU_HOURS=6`** | | | **~$270** | + +The throughput behind the GPU estimate is a guess, about 3K tok/s with +reference prefill plus a Python expert loop. Check it on the first sessions. If +`tok_per_s` is under 1,000 after the first 3 sessions, stop and rethink. At +that rate the teacher pass alone takes more than an hour. + +## Steps + +Only you can do the steps marked **(you)**. I do not enter credentials or +launch pods. + +1. **(you)** In RunPod: + - Set an account spend limit at or below $497. + - Add your SSH public key. + - Create a **1.5 TB network volume** in a data center that offers 8×H200. +2. **(you)** Start a cheap **CPU pod** with the volume at `/workspace` and an + exposed SSH port. +3. Copy the kit onto the volume. It lives only in local commits, so do not push: + + ```bash + git archive HEAD | ssh -p PORT root@HOST 'mkdir -p /workspace/keep && tar x -C /workspace/keep' + ``` + +4. On the CPU pod: + + ```bash + bash /workspace/keep/scripts/runpod/v41_reap/pod.sh fetch + ``` + + ```bash + bash /workspace/keep/scripts/runpod/v41_reap/pod.sh pack + ``` + + - **Check the fetch:** it must end with `fetch verified`, and the printed + model.py sha256 must be `e5decd76…8b8d2d`. + - **Check the pack:** read `/workspace/pack/manifest.json` and confirm every + domain reached its budget. A source that falls short shows up in its + `filtered`, `off_domain` or `non_english` counts. + + Then stop the CPU pod. +5. **(you)** Start an **8×H200 pod** with the same volume. Pick a PyTorch 2.10+ + / CUDA 12.8+ image and **at least 1 TB of system RAM**, because `convert.py` + holds the whole checkpoint in memory. +6. On the GPU pod, run inside `tmux`, since the pod stops itself when the stage + ends or at the limit: + + ```bash + MAX_GPU_HOURS=6 bash /workspace/keep/scripts/runpod/v41_reap/pod.sh gpu + ``` + + - **Watch the run:** follow `/workspace/out/progress.jsonl` for the first + sessions. + - **Results:** `/workspace/out/candidate-scores.json` holds the per-domain + KL, NLL change and top-1 agreement for N = 256/224/192/160. + - **Watchdog:** it calls `runpodctl stop pod`. If that fails it logs the + failure, so also keep the spend limit from step 1 in place. +7. **Choose N** from the scores. Take the smallest N whose worst domain stays + close to N=256, and which also fits the Mac's disk: 423 GB free, minus + about 30 GB of stats and logits. +8. **(you)** Start a CPU pod on the volume again, then run: + + ```bash + bash /workspace/keep/scripts/runpod/v41_reap/pod.sh prune 192 + ``` + +9. Pull the results to the Mac. Run each transfer as its own command. + + The pruned checkpoint: + + ```bash + rsync -avP --partial -e "ssh -p PORT" root@HOST:/workspace/v41-reap192/ ~/keep-artifacts/v41-reap192/ + ``` + + The stats, logits and pack: + + ```bash + rsync -avP --partial -e "ssh -p PORT" root@HOST:/workspace/out/ ~/keep-artifacts/v41-runpod-out/ + ``` + + ```bash + rsync -avP --partial -e "ssh -p PORT" root@HOST:/workspace/pack/ ~/keep-artifacts/v41-pack/ + ``` + + Then verify the checkpoint: + + ```bash + cd ~/keep-artifacts/v41-reap192 && shasum -a 256 -c SHA256SUMS --quiet + ``` + +10. **(you)** Delete the network volume once the Mac copies verify. + +## Notes + +- **Survivors:** `reap-select` picks them from calibration stats only. The + per-layer Jaccard against frequency is in `reap-select-N.json`; a value of + 0.9 or more means REAP added little. +- **Masked student pass:** it sets the router bias of pruned experts to -inf, + which equals slicing them (`tests/test_reap.py`). One converted checkpoint + therefore scores every N. +- **Eval sessions carry REAP stats and routes but no imatrix**, so the + per-split saliency can be compared later. +- **Sizes:** routed experts are about 289 GB of the 510 GB source, roughly + 0.75 GB per expert slot across the 40 layers. Engram (~196 GB) is untouched + and is most of what is left after pruning. diff --git a/docs/research/2026-11-05-v41-reap-calibration-data.md b/docs/research/2026-11-05-v41-reap-calibration-data.md new file mode 100644 index 0000000..eb5f5cd --- /dev/null +++ b/docs/research/2026-11-05-v41-reap-calibration-data.md @@ -0,0 +1,327 @@ +# Calibration and held-out data for REAP pruning of DeepSeek-V4.1-Flash-1017 + +Date: 2026-11-05. Scope: web research only. Every Hugging Face dataset id below +was checked against `https://huggingface.co/api/datasets/` on 2026-11-05/06 +unless it is marked **UNVERIFIED** or is hosted elsewhere. + +## 0. Target model facts (from the HF config, verified) + +`deepseek-ai/DeepSeek-V4.1-Flash-1017` (sha `1bae0e10...`, MIT, modified +2026-10-17): 40 layers, `hidden_size` 5120, `moe_intermediate_size` 2304, 384 +routed experts plus 1 shared expert, top-6, `sqrtsoftplus` scoring with +`noaux_tc`, FP8 dense weights with FP4 experts (block 32x32, ue8m0 scales), and +a 64K native context YaRN-scaled to 1M. The config also carries 3 MTP/"dspark" +layers that have their own 128-expert MoE. + +Implications for calibration: + +- An expert gets about 6/384 = 1.56% of tokens on average, so 3M calibration + tokens give about 47K routed tokens per expert per layer. Rare experts can get + less than 10% of that. +- The second moment of an expert's gate/up input is a 5120x5120 matrix, and + down_proj's input is 2304-dim. A well-conditioned estimate needs many more + tokens than d_in for each expert. Dense 2-bit methods use 25-35M tokens per + model (QuIP#: 6144 x 4096; QTIP Llama-3: 4096 x 8192) [Q1, Q2]. + +## 1. State of the art: calibration data for MoE expert pruning and PTQ + +### 1.1 What REAP itself did (arXiv 2510.13999, ICLR 2026) [R1, R2] + +- **Models up to about 110B:** 1,024 samples packed to 2,048 tokens. +- **Models of about 110B and up** (Qwen3-Coder-480B, Kimi-K2): 12,228 samples + with max length 16,384, no truncation and no packing. The mix includes + evol-codealpaca, `Salesforce/xlam-function-calling-60k`, and + `SWE-bench/SWE-smith-trajectories`. +- **Reasoning models** add 12,288 samples from `open-r1/Mixture-of-Thoughts` + (4,096 each from code, math and science). +- **Repo composite recipe** (24,576 samples, max 16K): 4,096 evol-codealpaca, + 12,288 Mixture-of-Thoughts, 4,096 xLAM, and 4,096 from the SWE-smith `tool` + split. It uses `dataset:samples` syntax. +- **Chat formatting:** the default chat template is applied, and `` is + closed for hybrid models. +- **Domain matters a lot for generative tasks.** Calibrating on C4 made code + generation "collapse": many pruned models emitted incoherent text and scored + 0%. On multiple-choice tasks C4 was equal or slightly better, which is why MC + benchmarks hide the damage. Domain-matched calibration helps most on + fine-grained MoEs (Qwen3-30B, ERNIE). A combined code + writing + math set gave + the best all-round retention for REAP. +- Cerebras GLM-4.6 REAP cards list the same three sources (evol-codealpaca, + xLAM, SWE-smith-trajectories) [R3]. + +### 1.2 Later work (2025-2026) + +| Paper | Calibration finding relevant to us | +|---|---| +| **EAN / MC-Suite**, Jaiswal et al., arXiv 2504.05586 [P1] | Among 16 criteria, the expert activation norm (EAN) was strongest. Expert dropping mainly hurts instruction following, and k-shot examples or SFT recover it. | +| **EASY-EP**, NeurIPS 2025, arXiv 2504.06792 [P2] | On DeepSeek-R1 and V3: 25 in-domain demonstrations recover about 99% of critical experts, and 5 recover over 90%. Input plus the model's **own generated output** beat inputs only, outputs only, inputs plus gold answer, and pretraining/CommonCrawl text (CommonCrawl scored about 0 on AIME). Mixed-domain pruning averages normalized per-domain scores. | +| **MoEQuant**, ICML 2025, arXiv 2505.03804 [P3] | Standard calibration sets give long-tailed expert coverage, so under-routed experts get too few calibration tokens and large quantization error. Expert-balanced self-sampling, where the model generates its own data, gave more than 10 points on HumanEval for DeepSeekMoE-16B at 4-bit. | +| **AIMER**, arXiv 2603.18492 (Apr 2026) [P4] | For REAP on Qwen3-30B at 50%, results were "highly sensitive to calibration set size even with the same corpus" across 0.5M to 2.1M C4 tokens. Baselines used 4.2M C4 tokens. REAP calibration took 0.75 to 3 hours on 7-30B models. | +| **HOPE**, Tseng et al. (AWS), arXiv 2609.18916 (Sep 2026) [P5] | Second-order expert pruning (REAP is its first-order special case). Calibrated on all 111K Evol-CodeAlpaca prompts, or on full **SWE-bench Verified agent trajectories** (tool calls plus observations, truncated at 100K tokens) generated by the unpruned model. REAP needed about 8K prompts to reach 99% prune-set stability. Calibration-set Jaccard between the two sets was about 0.6. On SWE-Bench Pro, the agentic calibration scored higher for REAP **at every pruning rate**, and for HOPE at 50%. | +| **Robust calibration for domain-general pruning**, arXiv 2610.15578 (Oct 2026) [P6] | On gpt-oss-20b, a default of 64 sequences was enough, and scaling beyond small sizes gave little gain. English-only calibration transferred across 7 languages. Diverse pretraining-like data matched test-distribution data. Caveat: code was **not** evaluated and results are preliminary. | +| **Generic TB-Coverage**, arXiv 2607.01710 [P7] | Profiles experts per corpus and keeps a coverage budget per corpus before the global mask. Generic corpora only; it beat REAP on zero-shot MC. | +| **OptEP**, arXiv 2610.27123 [P8] | Reconstruction-optimal subset selection with task-domain calibration. Its gains are largest under aggressive pruning. | +| **Post-compression adjustment**, arXiv 2609.06076 [P9] | 3,000 C4 examples and 1 epoch of fine-tuning recover 37% of the pruning gap. This is relevant if we fine-tune after REAP. | +| **Self-calibration**, Williams et al., NAACL 2025, arXiv 2410.17170 [P10] | Model-generated calibration (BOS-conditioned, temperature 1.0) is competitive with real data and often better for pruning. Quantization was less sensitive to calibration source. Self-generated text is less diverse. | +| **Beware of Calibration Data**, Ji et al., ICLR 2025, arXiv 2410.17711 [P11] | At high sparsity, calibration data can matter more than the pruning method. Self-generated data with a perplexity filter helps. | +| **On the Impact of Calibration Data in PTQ and Pruning**, Williams & Aletras, ACL 2024, arXiv 2311.09755 [P12] | Calibration source shifts downstream results substantially, more so for pruning than for quantization. | + +### 1.3 Synthesis + +1. **Domain-specific calibration is mandatory for code and agentic use.** REAP + saw C4 collapse code output, and EASY-EP saw CommonCrawl collapse reasoning. + MC-only evals hide this, so held-out evals must be generative or logit-based + on the target domains. +2. **Agentic multi-turn trajectories should be the largest slice.** HOPE is the + only direct comparison. There, SWE-trajectory calibration beat + Evol-CodeAlpaca for REAP on SWE-Bench Pro at every rate tested, and the two + sets choose about 40% different experts. REAP's own large-model and agentic + checkpoints added SWE-smith tool trajectories. The evidence is suggestive, not + overwhelming: some lower-rate cells favored plain code for other criteria. So + keep a plain-code slice too. +3. **Self-generated data beats human data when it is "prompt + the model's own + response".** EASY-EP (on DeepSeek-V3 and R1, our closest architecture + family), MoEQuant, Self-calibration and Ji et al. all agree. The reason is + that the router sees the distribution it will see at inference. HOPE also + generated its agentic traces with the unpruned model. For agent traces, where + observations come from an environment, the practical proxy is traces from a + sibling model. `nvidia/Open-SWE-Traces` v1.1 contains about 50K OpenHands + trajectories from **DeepSeek-V4-Flash-0731**. +4. **Size.** Pruning decisions stabilize quickly (64 sequences in [P6], 25 shots + per domain in [P2]), but REAP with fine-grained experts shows size + sensitivity between 0.5M and 2.1M tokens [P4]. Production REAP runs use + 12K-25K samples at 16K max, which is tens of millions of tokens. With 384 + experts, the binding constraint is **per-expert coverage**, both for REAP + saliency means and especially for the 2-bit second moments. A 2-4M token + budget is fine for saliency ranking but is at the low end for per-expert + Hessians. +5. **Sequence length.** REAP uses 16,384 for large models with no packing, and + HOPE uses up to 100K for agent traces. Agentic routing depends on deep + context (tool outputs and file contents late in the trajectory), so + 2,048-token packing under-represents it. + +## 2. Candidate datasets per domain + +Legend: **V** = verified via the HF API on this date. Size is approximate. A +"messages" format means role-tagged multi-turn chat. Datasets marked as +benchmarks are avoided as calibration on purpose. + +### 2.1 Agentic coding: Python, TypeScript, Rust (SWE-style tool loops) + +| Repo id | V | License | Size | Format | Updated | Notes and contamination | +|---|---|---|---|---|---|---| +| `nvidia/Open-SWE-Traces` (config v1.1, split `openhands`, path `data/openhands/deepseek_v4_flash_0731/`) | V | CC-BY-4.0 | v1.0 151K, v1.1 263K, plus v1.2/v2.0 trajectories; DSV4-Flash subset ~3.8 GB parquet | `messages` (system/user/assistant/tool) + `tools` JSON + `language` + `resolved` + patches | 2026-10-28 | **Top pick.** Tasks come from SWE-rebench-V2 and Scale-SWE. v1.0 counts: Python 34K, TypeScript 27K, Rust 17K. Includes **DeepSeek-V4-Flash-0731** trajectories, the closest public data to self-generated. SWE-rebench-V2 excludes SWE-bench repos but overlaps SWE-bench Multilingual (12 tasks) and Multi-SWE-bench (83 tasks); exclude those IDs ([C1], agentleak). | +| `SWE-bench/SWE-smith-trajectories` | V | MIT | ~50K | `messages`, SWE-agent tool calls | 2025-07-19 | Used by REAP. Python-only synthetic bugs. Moderately used; good as a secondary harness for diversity. | +| `nebius/SWE-rebench-openhands-trajectories` | V | CC-BY-4.0 | 10K-100K | `messages` (OpenHands) | 2025-12-27 | Python. Overlaps SWE-rebench tasks in Open-SWE-Traces, so dedupe by `instance_id`. | +| `togethercomputer/CoderForge-Preview` | V | **no repo license** (per-row repo license only) | 258K trajectories, ~93 GB | OpenHands `messages` + `tools` | 2026-02-26 | See the section 4 vetting. Python-only, Qwen3-Coder-480B teacher. Better as eval. | +| `nvidia/SWE-Hero-openhands-trajectories` | V | CC-BY-4.0 | 10K-100K | OpenHands `messages` | 2026-05-08 | Alternative or overflow. | + +Plain code, real repository files and diffs (to cover code reading and writing +outside agent loops): + +| Repo id | V | License | Size | Format | Updated | Notes | +|---|---|---|---|---|---|---| +| `nebius/SWE-rebench-V2-PRs` | V | CC-BY-4.0 (per-repo SPDX column) | 126K real PRs, multi-language (Python, TS, Rust, Go...) | issue + PR description + gold `patch` + `test_patch` | 2026-08-18 | Real human commits, 2025-2026 heavy. Render as "issue -> diff". Same benchmark-overlap caveat as above. | +| `common-pile/stackv2_edu_filtered` | V | per-file open licenses (Common Pile) | 10M-100M files | single `text` + rich `metadata` (licenses, extension) | 2025-06-06 | Openly licensed, edu-filtered Stack v2 files. Filter by extension `.py/.rs/.ts/.tsx` and prefer `committer_date` >= 2023. | +| `HuggingFaceTB/stack-edu` (configs `Python`, `Rust`, `TypeScript`) | V | per-file (Stack v2 terms) | 100M+ files | ids/blobs; content fetched from Software Heritage S3 | 2025-03-20 | Use only if the Common Pile filter is too thin for Rust. Needs a SWH download step. | +| `SWE-bench/SWE-smith-rs`, `SWE-bench/SWE-smith-ts` | V | not stated in card | 1K-10K tasks each | task instances (no trajectories) | 2026-02 | Good **eval** seeds for Rust/TS agent rollouts by the teacher. | + +### 2.2 Bash / shell / terminal agents + +| Repo id | V | License | Size | Format | Updated | Notes | +|---|---|---|---|---|---|---| +| `nvidia/Nemotron-Terminal-Corpus` | V | CC-BY-4.0 | ~366K trajectories (`dataset_adapters` ~226K, `skill_based_*` ~140K) | terminus-2 `messages` (keystrokes / command JSON + terminal output) | 2026-02-27 | **Top pick.** Synthetic tasks from a primitive-skill taxonomy; the target benchmark is Terminal-Bench 2.0. Prefer `skill_based_*` for shell skills. | +| `open-thoughts/AgentTrove` | V | Apache-2.0 | 1.7M rows from 219 sources | terminus-2 `messages` + `original_source` + `original_teacher` + `reward` | 2026-05-07 | Filter `original_source` (e.g. `nl2bash`, `staqc`, `freelancer`) and `reward==1`. Teachers include GLM-4.6/4.7, Kimi K2, MiniMax, and GPT-5.1-nano/5-mini. Flag the OpenAI-generated rows (ToS) and drop them if that is a concern. | +| `open-thoughts/OpenThoughts-Agent-v1-SFT` | V | Apache-2.0 | 10K-100K | terminus-2 `messages` | 2026-01-27 | Smaller curated subset. Overlaps AgentTrove, so pick one. | +| `nvidia/Nemotron-RL-Agentic-Terminal-Pivot-v1` | V | CC-BY-4.0 | 10K-100K | RL prompts and environments | 2026-08-28 | Useful as **eval seeds**: roll out with the teacher. | + +### 2.3 General tool use (function calling, multi-turn tool loops) + +| Repo id | V | License | Size | Format | Updated | Notes | +|---|---|---|---|---|---|---| +| `Agent-Ark/Toucan-1.5M` | V | Apache-2.0 | 1.5M trajectories over 495 real MCP servers and 2,000+ tools | `messages` (Hermes tool format) + `subset` (`multi-turn`, `irrelevant`, ...) + LLM-judge quality | 2025-10-04 | **Top pick.** Real tool executions with parallel and sequential calls. Configs `Kimi-K2`, `Qwen3`, `OSS`, `SFT`. Use `multi-turn` with high judge scores. Targets BFCL/MCP-Universe. | +| `nvidia/Nemotron-Agentic-v1` | V | CC-BY-4.0 | ~? (multi-subset) | multi-turn user/agent/tool simulation | 2025-12-15 | Qwen3-235B / GPT-OSS generated and judge-filtered. Persona-seeded. | +| `nvidia/Nemotron-SFT-Agentic-v4` | V | OpenMDW-1.1 | multi-subset | `tool_calling` + `knowledge_agent` traces | 2026-10-15 | Newest; generated by Qwen3.5-397B and GLM-5.2. Check that OpenMDW is acceptable. | +| `Nanbeige/ToolMind` | V | Apache-2.0 | 100K-1M | multi-turn tool-use `messages` | 2026-01-09 | Alternative. | +| `Salesforce/APIGen-MT-5k` | V | **CC-BY-NC-4.0** | 5K | multi-turn (tau-bench-like) | 2025-10-10 | Non-commercial; **eval-only** at most. | +| `Salesforce/xlam-function-calling-60k` | V | CC-BY-4.0 (gated auto) | 60K single-turn | query + tools + answer | 2025-01-24 | Used by REAP. Single-turn and widely used, so give it a small share at most. | + +### 2.4 Networking + +Hugging Face has no strong, permissive networking corpus. Recommended sources: + +| Source | V | License | Size | Format | Notes | +|---|---|---|---|---|---| +| **IETF RFCs** from rfc-editor.org (`https://www.rfc-editor.org/rfc/rfcNNNN.txt`, index `rfc-index.xml`) | URL verified (HTTP 200); not HF | IETF Trust Legal Provisions (RFCs may be copied and redistributed in full) | ~9,900 RFCs | plain text | **Top pick.** Use recent RFCs (RFC 9000+, 2021-2026: QUIC, HTTP/3, TCP 9293, TLS, BGP, DNS) and hold out by RFC number. The bulk `RFC-all.tar.gz` path returned 404, so fetch per-RFC. | +| `common-pile/stackexchange_filtered` (filter `metadata.site` in `networkengineering.stackexchange.com`, `serverfault.com`, `unix.stackexchange.com`, `superuser.com`) | V | CC-BY-SA (per-doc `license`) | 31M docs total | question + answers + comments as one `text` | Real human operational Q&A. Filter `created` >= 2022 to reduce memorized content. | +| `jiebi/RFCAlign` | V | MIT | 1M-10M | synthetic Q/A from IETF mailing lists (Llama/Qwen) | Synthetic retrieval data. Low priority. | +| `nvidia/Nemotron-Terminal-Corpus` (network tasks inside) | V | CC-BY-4.0 | n/a | n/a | Adds hands-on `ip`/`ss`/`curl`/`dig` tool loops. | + +### 2.5 Cybersecurity + +| Repo id | V | License | Size | Format | Updated | Notes | +|---|---|---|---|---|---|---| +| `trendmicro-ailab/Primus-FineWeb` | V | ODC-BY (gated: accept terms) | 2.57B tokens | single `text` web documents | 2025-08-09 | **Top pick** for prose. Classifier-filtered FineWeb; snapshots through CC-MAIN-2024-10. | +| `CIRCL/vulnerability-scores` | V | CC-BY-4.0 | 837K records: CVE 5, GHSA, CSAF Red Hat/CISA/Cisco, PySec | title + description + CPEs + CVSS + patch URL | 2026-10-29 | Real advisories, continuously updated. Has **group-aware train/val/test splits** (no CVE/GHSA leakage), which makes it ideal for a disjoint held-out set. | +| `github/advisory-database` (GitHub, not HF) | V (GitHub API) | CC-BY-4.0 | ~300K+ OSV JSON | JSON advisories | pushed 2026-11-06 | Overlaps CIRCL's `github` source; use one. | +| `PentesterFlow/OffsecCyber` | V (page) | Apache-2.0 | 60.5K | `messages` (some with `tools`) | 2026 | Mixed: about 51% Qwen-synthetic, plus NVD, HackerOne and PortSwigger. Web-app focus. Secondary. | +| `common-pile/stackexchange_filtered` (`security.stackexchange.com`, `reverseengineering`) | V | CC-BY-SA | n/a | Q&A text | n/a | Real human Q&A. | + +Avoid: `Vyber07/cyber-security` (see section 4), `own2pwn-fr/ctfhoard-corpus` +(245 GB, no card, no license), and CTF benchmark sets (`penfever/*-harbor`, +`cvebench`, intercode-ctf), which are evals. + +### 2.6 Business documents + +| Repo id | V | License | Size | Format | Updated | Notes | +|---|---|---|---|---|---|---| +| `HuggingFaceFW/finepdfs` (config `eng_Latn`) | V | ODC-BY | 100M+ PDFs (all langs) | `text` + `url` + quality scores | 2026-04-03 | **Top pick.** Real PDFs (agendas, reports, policies, contracts). Filter by URL/domain and keywords (invoice, policy, minutes, proposal, SOW, quarterly) and by `ocr_quality_scores`. Hold out by URL domain. | +| SEC EDGAR full text (sec.gov, public domain), or `juand-r/ai-sec-10k-filings-since-2020` | V | HF card has no license; EDGAR itself is public domain | 1B-10B rows | raw SEC SGML text | 2026-05-31 | Take MD&A and risk-factor sections from 2023-2026 10-K/10-Q filings. Strip SGML. | +| `corbt/enron-emails` | V | no license field (Enron corpus released via FERC) | ~500K emails | structured email fields | 2025-04-07 | Real business correspondence, but dated (2001). Small share. | +| `common-pile/regulations_filtered` / `common-pile/usgpo_filtered` | V | public domain / open | 100K-10M | `text` | 2025-06-06 | Government and regulatory documents. Optional. | +| `nvidia/Nemotron-SFT-Finance-Document-Grounded-Reasoning-v1` | V | OpenMDW-1.1 | n/a | document-grounded `messages` | 2026-10-14 | Synthetic. Optional for document-QA style. | + +### 2.7 English (general instruction and prose) + +| Repo id | V | License | Size | Format | Updated | Notes | +|---|---|---|---|---|---|---| +| `allenai/WildChat-4.8M` | V | ODC-BY | 4.8M real user conversations | `conversation` messages + `language` | 2025-08-11 | **Top pick** for real prompts. Filter `language=="English"`, drop toxic and redacted rows, **use the user prompts and regenerate responses with V4.1-Flash** (EASY-EP style). Not a benchmark. | +| `HuggingFaceFW/finewiki` (config `en`) | V | CC-BY-SA-4.0 / GFDL | 10M-100M | `text` | 2025-10-22 | Clean 2025 Wikipedia. | +| `allenai/dolma3.5_pool` | V | ODC-BY | huge | `text` | 2026-07-10 | Pretraining-like diversity ([P6] finds this is enough for domain-general retention). | +| `nvidia/Nemotron-SFT-Instruction-Following-Chat-v3` | V | mixed (ODC-BY / CC-BY / Apache / other) | 100K-1M | `messages` | 2026-06-04 | Synthetic. Alternative prompt source. | + +## 3. Recommended mix + +### 3.1 Principles + +- **Format.** Render everything with V4.1-Flash's own chat template and tool-call + format. Convert Hermes/OpenHands/terminus tool calls into the model's native + tool schema; otherwise the router sees foreign control tokens. Match the + thinking mode to deployment. If both modes are used, take about 60% thinking + and 40% non-thinking. +- **Self-generation.** For prompt-type domains (English, tool use, cyber Q&A, + networking Q&A, business-doc QA), keep the human or source prompts and + **regenerate the assistant turns with V4.1-Flash itself**. You already run the + teacher forward pass, so the cost is generation time only. For agentic traces, + prefer the DeepSeek-V4-Flash-0731 trajectories in Open-SWE-Traces. Keep raw + documents (RFCs, code files, advisories, filings) human-written: in agent use + they arrive as tool observations the router must handle. +- **Sequence length.** Use **max 16,384** tokens, with no packing across + conversations, as REAP does for its large models. For agent trajectories + longer than 16K, take message-aligned windows that keep the system prompt and + tools header plus a random contiguous span of later turns, not just the first + 16K. Add a **long-context slice of about 10% of tokens at 32K-64K** (the + model's native 64K pre-YaRN) from Open-SWE-Traces, so that long-horizon routing + is represented, as HOPE did with traces up to 100K. Pack short documents + (advisories, Q&A) to 4K-8K with EOS separators. +- **Coverage gate.** After the forward pass, log routed-token counts for each + (layer, expert). For REAP, require at least 1K routed tokens for every expert + you might keep. For the 2-bit second moments, require at least about 2 x d_in + (about 10K tokens for gate/up, about 5K for down) for every **kept** expert. + If any kept expert falls short, top up with more agentic traces, or with + MoEQuant-style self-sampling steered toward under-covered experts, before + computing Hessians. +- **Seeds.** As REAP did, check the stability of the prune set across 2 + calibration seeds or subsamples (Jaccard). HOPE found about 0.6 between + different domain sets. + +### 3.2 Calibration split (~3.2M tokens) + +| Domain | Tokens | Sources (in priority order) | +|---|---|---| +| Agentic coding (Python, TS, Rust) | 1,000K | Open-SWE-Traces v1.1 DeepSeek-V4-Flash-0731 OpenHands: Python 400K / TS 300K / Rust 300K. Fill Rust/TS gaps from v1.0/v1.2 or SWE-smith-trajectories. Include the 32K-64K long windows here. | +| Plain code + real diffs | 400K | SWE-rebench-V2-PRs (issue -> patch) 200K, plus stackv2_edu_filtered `.py/.rs/.ts` files 200K, split evenly across the three languages. | +| Bash / terminal agent | 400K | Nemotron-Terminal-Corpus `skill_based_*` 250K, plus AgentTrove (`nl2bash`/`staqc`, reward 1, non-OpenAI teachers) 150K. | +| General tool use | 450K | Toucan-1.5M `multi-turn` 250K, plus Nemotron-SFT-Agentic-v4 / Nemotron-Agentic-v1 150K, plus xLAM single-turn 50K. | +| Cybersecurity | 300K | Primus-FineWeb 150K, CIRCL vulnerability-scores (train split) 75K, and security.stackexchange 75K (prompts with self-generated answers). | +| Networking | 250K | Recent RFCs 150K, plus networkengineering/serverfault/unix StackExchange 100K. | +| Business documents | 250K | finepdfs eng_Latn business-filtered 150K, SEC 10-K/10-Q sections 2023-2026 60K, Enron 40K. | +| English general | 150K | WildChat English prompts with V4.1-Flash responses 100K, plus finewiki en 50K. | +| **Total** | **~3.2M** | About 60% agentic or tool-loop tokens, which matches the target use. | + +Rationale: + +- Agentic and tool-loop data gets the largest share because HOPE showed + agentic calibration helps REAP on SWE-Bench Pro at every rate, and REAP's + agentic checkpoints use it. +- Every target domain gets at least 150K tokens, because REAP and EASY-EP show + that a domain absent from calibration loses its experts. EASY-EP's 25-shot + result suggests 150K tokens is far above the minimum for ranking. +- The total stays within 2-4M because [P6] and [P2] show ranking saturates + early. AIMER's 0.5M-2.1M sensitivity argues against going below about 2M. +- Before quantization, consider growing the agentic slice to about 8-10M tokens + for Hessian accuracy alone (QTIP/QuIP# scale). That data need not enter the + REAP saliency pass. + +### 3.3 Held-out eval split (disjoint), for teacher top-k logits + +Use about 40-60K tokens per domain (~450K total, 100-300 sequences per domain), +with the same template and lengths. Make each split disjoint as follows: + +- **Agentic coding:** disjoint **by repository** from calibration, not just by + instance. Use Open-SWE-Traces repos held out, plus CoderForge `filtered_reward1` + Python repos not in calibration. CoderForge's different teacher and harness + also give a robustness check. Seed Rust/TS agent rollouts from SWE-smith-rs/ts + tasks generated by the teacher. +- **Bash:** a different `skill` taxonomy branch, or a different AgentTrove + `original_source`. +- **Tool use:** held-out MCP servers in Toucan (split by `target_tools` server). +- **Cyber:** CIRCL's own `test` split, which is group-split by CVE/alias. +- **Networking:** a held-out set of RFC numbers, plus StackExchange questions + created after the calibration cutoff date. +- **Business:** held-out URL domains in finepdfs, plus different SEC filers. +- **English:** held-out WildChat conversation ids. + +Also report per-domain KL against the teacher's top-k, not only accuracy. The +REAP finding that MC accuracy hides generative collapse applies here. + +## 4. Vetting of five candidates Jack suggested + +Revision SHAs are pinned as of 2026-11-05/06. + +| Field | `togethercomputer/CoderForge-Preview` | `nvidia/Open-SWE-Traces` | `Vyber07/cyber-security` | `MoreThought/Fable-5.1-Max-Reasoning-Filtered-10000x` | `Axion1-0/axion-training-data` | +|---|---|---|---|---|---| +| Pinned SHA | `060fca96cf723b2ebab3181e9e59fafd273df3cb` | `4fa4c2ae9afc6b397bc50391d43450c79993c138` | `bc7a780571cb67d482250649ab573908b9557873` | `4904ca0c09044f447aff16e4f66b6f2f53c9f702` | `668831c96e7c3d974bc5c414329d508277f23663` | +| License / gated | **No repo-level license** in the card or tags. Each row has a `license` field holding the source repo's SPDX id (e.g. Apache-2.0). Not gated. | CC-BY-4.0, not gated, "commercial/non-commercial use" | Card says Apache-2.0. **Gated (auto)**: files need you to share HF account info. We did not accept the gate, so contents are unverified. | Card says Apache-2.0, not gated. The uploader's license cannot override the generating model's terms (see ToS). | Apache-2.0, not gated | +| Size / format | 258K trajectories (155K with reward 1), ~93 GB raw. Avg **41.4K tokens** per trajectory (P99 91K). `messages` JSON string + `tools` JSON + `reward` + `finish_reason` + Docker `image`. Also a Qwen-tokenized config. | v1.0 151K, v1.1 263K, v1.2/v2.0 more. Parquet with `messages` (system/user/assistant/tool), `tools`, `language`, `resolved`, gold and model patches. | Card claims 16 GB / 507 files in one place and ~95 GB in another; the Hub tag says <1K rows. The top level of the tree shows 1,000 entries, ~13 GB. ShareGPT, instruction/IO and "metadata-only" JSON schemas mixed; card says only 394/507 files are valid JSON. Includes a stray `.openclaude__settings.local.json`. | 10K traces (5K `train`, 10K `full`), 1.97 GB, claimed ~500M tokens (~50K per trace). `messages` + `tools` (`bash`, `read_file`, `write_file`, `search`, `list_dir`). | 18,363 SFT trajectories (stage1 10,819, stage2 7,544) + 484 RL tasks, ~533 MB. `messages`; tool responses are in `user` turns. | +| Multi-turn / tool calls | **Yes, agentic.** OpenHands v0.52.1 tools: `execute_bash`, `str_replace_editor` (file edits), `think`, `finish`. Up to 100 steps in Docker; real executions. | **Yes, agentic.** OpenHands (`execute_bash`, editor), SWE-agent, mini-swe-agent, Claude-Code and OpenCode harnesses. Real executions in SWE-rebench-V2 / Scale-SWE containers. | Some ShareGPT multi-turn chat, per the card. No tool calls described. | Superficially yes (avg 40-100 messages), but **1,370 of 1,372 tool-call argument strings in our sample are placeholders** (`{"path": ""}`, `{"command": ""}`). Tool names look remapped: a `search` call returns "Command running in background..." output. | Yes: search, web visit and Python tool loops (stage2); Python only (stage1). | +| Language | English (prompts are GitHub issues; a few non-English issues are possible) | English, with the same caveat. Run langid on `problem_statement` and drop non-English. | Card says English (unverified) | English | English | +| Jack's domains covered | Python agentic coding and bash (inside the loop). No Rust or TS (blog and prompt template are Python-repo only). | **Python, TypeScript, Rust** agentic coding (plus Go/JS/Java/PHP), bash inside the loop | Cybersecurity, possibly networking (claimed) | Claimed coding and bash, but the visible prompts are the uploader's own data-pipeline "seed authoring" tasks | General tool use, but in scientific research (biology, chemistry, KEGG/UniProt). Not coding. | +| Provenance | Model-generated by **Qwen3-Coder-480B** (open weights, Apache-2.0). No commercial-model ToS issue. Tasks: R2E-Gym (4.2K), SWE-Smith (37.2K), SWE-Rebench (9.8K), 1,655 repos. | Model-generated by **MiniMax-M2.5, Qwen3.5-122B, Qwen3.6-27B, DeepSeek-V4-Flash-0731, Qwen3.8-27B, Qwen3.8-Flash-Next**, all open-weight. No commercial-model ToS issue. The "claude-code" split uses the Claude Code harness driven by Qwen3.8 models, not Claude outputs. | Undocumented: no statement of human vs model generation, no model named, sources (HackerOne, HTB, TryHackMe, PortSwigger) with no licensing provenance. | **Distilled from "Fable 5.1", an Anthropic Claude-family model** ("Claude Fable 5" appears in Axion's comparison table). Traces were captured from **Claude Code sessions**: paths `~/.claude/projects/...` and `/tmp/claude-1000/...` appear in tool outputs. | Model-generated by an **unnamed** LLM solver. GPT-5.5 was used only to verify reference answers. The student is Qwen3.5-122B. If the solver was a commercial API, the ToS status is unknown. | +| Contamination risk | Low for SWE-bench Verified: the authors excluded tasks matching SWE-bench Verified by repo + base commit or problem statement. Not checked against SWE-bench Multilingual or Multi-SWE-bench. SWE-Smith repos are synthetic-bug repos. | SWE-rebench-V2 excludes SWE-bench repos but overlaps SWE-bench Multilingual (12 tasks, 15 repos) and Multi-SWE-bench (83 tasks, 16 repos) per agentleak; exclude those ids. Scale-SWE (Python, CC-BY-4.0) reports SWE-bench Verified results; check its repos against the 12 Verified repos. | Unknown; cannot inspect. PortSwigger/HTB lab writeups may mirror CTF benchmarks. | Unknown task sources. Not relevant given the reject. | No explicit SFT decontamination statement. RL tasks exclude held-out eval overlap. Science QA could overlap GPQA/HLE-style evals (not our targets). | +| Dedup / overlap with other picks | **High overlap** with Open-SWE-Traces (both draw on SWE-rebench) and with SWE-smith-trajectories (SWE-Smith tasks). If CoderForge supplies eval, the calibration split must exclude its repos (dedupe on repo + base commit, or `instance_id` parsed from `trajectory_id`). | Is the primary pick. Overlaps nebius/SWE-rebench-openhands-trajectories, SWE-rebench-V2-PRs and CoderForge at the task and repo level. | Likely overlaps PentesterFlow/OffsecCyber and NVD/CVE sources (unverifiable) | n/a | None with the other picks (different domain) | +| Recommendation | **Eval-only** (primary): a held-out Python agentic split from a different teacher and harness, measuring generalization beyond the DeepSeek-distribution calibration. Use `filtered_reward1` with repos disjoint from calibration, cut to message-aligned 16K-32K windows. Calibration use is acceptable only as overflow, after the repo-level license gap is accepted. | **Calibration: top pick** for Python/TS/Rust agentic coding. Use the v1.1 DeepSeek-V4-Flash-0731 OpenHands subset first, as the nearest-to-self data. Pin the SHA, filter by `language`, prefer `resolved==1` (keep some failures for realism), and exclude agentleak overlap ids. | **Reject.** Gated content we have not inspected; undocumented provenance and licensing of platform material; self-inconsistent card; an explicit "no restrictions" offensive scope with no safety review; tooling artifacts in the repo. Use Primus-FineWeb + CIRCL instead. | **Reject.** (1) Commercial-model **ToS problem**: Anthropic's terms restrict using outputs to build competing models, and an uploader's Apache-2.0 label cannot grant rights it does not hold. Calibrating and distilling a pruned model on it is at least a gray area. (2) Data defects: tool arguments replaced by placeholders, so the tokens are unrealistic for routing statistics. (3) Heavy duplication: only 5 distinct first-user prompts across the first 23 traces, and several near-identical traces, despite the dedup claim. (4) Prompts are meta "seed-authoring" tasks for the uploader's pipeline. | **Reject for calibration** (off-domain science research; non-native tool-response format in `user` turns; unnamed teacher). Optional **eval-only** as a small out-of-domain tool-loop probe, to watch for collateral damage to general tool use. Low priority. | + +Notes: + +- The Vyber07 card text above was read from the public page only; the files were + not downloaded because that needs the gate accepted. +- Fable statistics come from the first ~3 MB (23 traces) of `5k/train.jsonl`, + fetched with an HTTP range request and parsed as data. + +## 5. Sources + +- [R1] REAP paper: https://arxiv.org/abs/2510.13999 (HTML https://arxiv.org/html/2510.13999) +- [R2] REAP code and composite calibration recipe: https://github.com/CerebrasResearch/reap +- [R3] Cerebras REAP card: https://huggingface.co/cerebras/GLM-4.6-REAP-252B-A12B-FP8 +- [P1] EAN / MC-Suite: https://arxiv.org/abs/2504.05586 +- [P2] EASY-EP: https://arxiv.org/abs/2504.06792 (NeurIPS 2025) +- [P3] MoEQuant: https://arxiv.org/abs/2505.03804 +- [P4] AIMER: https://arxiv.org/html/2603.18492v2 +- [P5] HOPE: https://arxiv.org/abs/2609.18916 +- [P6] Robust calibration for domain-general MoE pruning: https://arxiv.org/abs/2610.15578 +- [P7] Generic TB-Coverage: https://arxiv.org/abs/2607.01710 +- [P8] OptEP: https://arxiv.org/abs/2610.27123 +- [P9] Post-compression adjustment: https://arxiv.org/abs/2609.06076 +- [P10] Self-calibration: https://arxiv.org/abs/2410.17170 +- [P11] Beware of Calibration Data: https://arxiv.org/abs/2410.17711 +- [P12] Calibration data in PTQ and pruning: https://arxiv.org/abs/2311.09755 +- [Q1] QuIP#: https://arxiv.org/abs/2402.04396 +- [Q2] QTIP: https://arxiv.org/abs/2406.11235 +- [C1] SWE-rebench-V2 benchmark overlap (agentleak): https://github.com/raimondasl/agentleak +- CoderForge blog: https://www.together.ai/blog/coderforge-preview +- Axion 1.0 paper: https://huggingface.co/papers/2611.03233 +- SWE-rebench-V2: https://arxiv.org/abs/2602.23866 +- Terminal-Corpus paper: https://arxiv.org/abs/2602.21193 +- Toucan: https://arxiv.org/abs/2510.01179 +- Primus: https://arxiv.org/abs/2502.11191 +- Dataset pages: `https://huggingface.co/datasets/` for every id above. diff --git a/scripts/runpod/v41_reap/build_pack.py b/scripts/runpod/v41_reap/build_pack.py new file mode 100644 index 0000000..487d0d4 --- /dev/null +++ b/scripts/runpod/v41_reap/build_pack.py @@ -0,0 +1,381 @@ +"""Build the V4.1 REAP calibration + held-out eval pack from public datasets. + +Mix and rationale: docs/research/2026-11-05-v41-reap-calibration-data.md +(section 3). English only, by policy: any record containing CJK, Hangul or +Kana characters, or more than 2% non-Latin letters, is dropped. + +Every chat source is rendered with V4.1's own ``encoding/encoding.py`` (tools +on the system message, DSML tool calls) and tokenized with the V4.1 +tokenizer. Raw documents go in as BOS + text. Records are capped at +``--max-tokens`` (hard truncation). Short documents are packed to about +``--pack-tokens`` with EOS separators. + +Split: each record's group key (repo, task, RFC number, URL host, CVE id, +...) is hashed, and one bucket in ten feeds the eval split until that +domain's eval budget is met, so calibration and eval never share a group. +Calibration records are grouped into sessions of up to ``--session-tokens``; +each eval record is its own session (one logits stream each). + +Run on the pod (a CPU pod is enough), not the Mac:: + + python scripts/runpod/v41_reap/build_pack.py --model-dir /workspace/v41-hf \\ + --out /workspace/pack + +Output: ``v41-pack.jsonl`` (collect.py's input) and ``manifest.json`` +(resolved dataset revisions, rows kept/dropped, tokens per domain and split). +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +import urllib.request +from collections import Counter, defaultdict +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +K = 1000 + +# fmt: off +#: name, dataset, load kwargs, adapter, domain (str, or a callable on the row), group-key field, {domain: (calib, eval) tokens} +SOURCES: list[dict[str, Any]] = [ + dict(name="open_swe_traces_dsv4", dataset="nvidia/Open-SWE-Traces", + load=dict(data_files="data/openhands/deepseek_v4_flash_0731/*/*.parquet", split="train"), + adapter="openai", domain=lambda r: f"agentic-{r['language']}", group="repo", + budget={"agentic-python": (400 * K, 50 * K), "agentic-typescript": (300 * K, 40 * K), "agentic-rust": (300 * K, 40 * K)}), + dict(name="swe_rebench_v2_prs", dataset="nebius/SWE-rebench-V2-PRs", load=dict(split="train"), + adapter="issue_patch", domain="code-diffs", group="repo", budget={"code-diffs": (400 * K, 50 * K)}), + # ponytail: no plain-file source. stackv2_edu_filtered's shards are grouped by language with no index, + # so finding .py/.rs/.ts means scanning most of it; the agent traces already show files through tool output. + dict(name="nemotron_terminal", dataset="nvidia/Nemotron-Terminal-Corpus", load=dict(name="skill_based_mixed", split="train"), + adapter="terminus", domain="bash", group="task", budget={"bash": (400 * K, 50 * K)}), + dict(name="toucan", dataset="Agent-Ark/Toucan-1.5M", load=dict(name="SFT", split="train"), + adapter="toucan", domain="tool-use", group="tools", budget={"tool-use": (450 * K, 60 * K)}), + dict(name="circl_vulns", dataset="CIRCL/vulnerability-scores", load=dict(split="train"), + adapter="advisory", domain="cyber", group="id", pack=True, budget={"cyber": (150 * K, 25 * K)}), + dict(name="stackexchange", dataset="common-pile/stackexchange_filtered", load=dict(split="train"), + adapter="stackexchange", domain=lambda r: _se_domain(r), group="url", pack=True, + budget={"cyber-qa": (150 * K, 25 * K), "networking-qa": (100 * K, 20 * K)}), + dict(name="rfcs", dataset="rfc-editor.org", load=dict(first=9000, last=9800), + adapter="rfc", domain="networking", group="rfc", budget={"networking": (150 * K, 25 * K)}), + dict(name="finepdfs_business", dataset="HuggingFaceFW/finepdfs", load=dict(name="eng_Latn", split="train"), + adapter="business_pdf", domain="business", group="url", group_host=True, budget={"business": (210 * K, 30 * K)}), + dict(name="enron", dataset="corbt/enron-emails", load=dict(split="train"), + adapter="email", domain="business-email", group="from", pack=True, budget={"business-email": (40 * K, 10 * K)}), + dict(name="wildchat_prompts", dataset="allenai/WildChat-4.8M", load=dict(split="train"), + adapter="wildchat", domain="english-chat", group="conversation_hash", budget={"english-chat": (50 * K, 15 * K)}), + dict(name="finewiki", dataset="HuggingFaceFW/finewiki", load=dict(name="en", split="train"), + adapter="text", domain="english-prose", group="id", budget={"english-prose": (100 * K, 15 * K)}), +] +# fmt: on + +SE_SITES = { + "cyber-qa": ("security.stackexchange.com", "reverseengineering.stackexchange.com"), + "networking-qa": ("networkengineering.stackexchange.com", "serverfault.com", "unix.stackexchange.com"), +} +BUSINESS_WORDS = re.compile( + r"\b(invoice|purchase order|statement of work|quarterly report|annual report|board minutes|meeting minutes|" + r"business plan|proposal|policy|procedure|memorandum|agreement|contract|budget|financial statements?)\b", + re.IGNORECASE, +) +NON_ENGLISH_SCRIPT = re.compile(r"[぀-ヿ㐀-䶿一-鿿가-힯豈-﫿ｦ-ﾟ]") + + +def is_english(text: str) -> bool: + """No CJK / Hangul / Kana at all, and at most 2% of letters outside Latin.""" + + if NON_ENGLISH_SCRIPT.search(text): + return False + letters = [c for c in text if c.isalpha()] + foreign = sum(1 for c in letters if ord(c) > 0x024F) + return not letters or foreign <= 0.02 * len(letters) + + +def _se_domain(row: dict[str, Any]) -> str | None: + site = (row.get("metadata") or {}).get("site") + return next((domain for domain, sites in SE_SITES.items() if site in sites), None) + + +# --------------------------------------------------------------------------- +# Adapters: row -> ("chat", messages) | ("doc", text) | None +# --------------------------------------------------------------------------- + + +def _json(value: Any) -> Any: + return json.loads(value) if isinstance(value, str) else value + + +def _split_think(content: str) -> tuple[str, str]: + match = re.match(r"\s*(.*?)\s*(.*)", content or "", re.DOTALL) + return (match.group(1).strip(), match.group(2)) if match else ("", content or "") + + +def adapt_openai(row: dict[str, Any]) -> tuple[str, Any] | None: + messages = [] + for m in _json(row["messages"]): + msg = {"role": m["role"], "content": m.get("content") or ""} + if m.get("reasoning_content"): + msg["reasoning_content"] = m["reasoning_content"] + if m.get("tool_calls"): + msg["tool_calls"] = _json(m["tool_calls"]) + if m["role"] == "tool" and m.get("tool_call_id"): + msg["tool_call_id"] = m["tool_call_id"] + messages.append(msg) + tools = [_json(t) for t in _json(row.get("tools") or [])] + if tools and messages and messages[0]["role"] == "system": + messages[0]["tools"] = tools + elif tools: + messages.insert(0, {"role": "system", "content": "", "tools": tools}) + return "chat", messages + + +def adapt_terminus(row: dict[str, Any]) -> tuple[str, Any] | None: + messages = [] + for m in row["conversations"]: + if m["role"] == "assistant": + reasoning, content = _split_think(m["content"]) + messages.append({"role": "assistant", "content": content, **({"reasoning_content": reasoning} if reasoning else {})}) + else: + messages.append({"role": m["role"], "content": m["content"]}) + return "chat", messages + + +def adapt_toucan(row: dict[str, Any]) -> tuple[str, Any] | None: + if "multi-turn" not in (row.get("subset_name") or ""): + return None + messages: list[dict[str, Any]] = [{"role": "system", "content": "", "tools": _json(row["tools"])}] + calls = 0 + for m in _json(row["messages"]): + role, content = m["role"], m.get("content") or "" + if role == "tool_call": + call = ast.literal_eval(content) if isinstance(content, str) else content + calls += 1 + if messages[-1]["role"] != "assistant": + messages.append({"role": "assistant", "content": ""}) + messages[-1].setdefault("tool_calls", []).append( + {"id": f"call_{calls}", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}} + ) + elif role == "tool_response": + messages.append({"role": "tool", "content": content, "tool_call_id": f"call_{calls}"}) + else: + messages.append({"role": role, "content": content}) + return "chat", messages + + +def adapt_issue_patch(row: dict[str, Any]) -> tuple[str, Any] | None: + issue = row.get("problem_statement") or row.get("pr_description") or "" + if not issue or not row.get("patch"): + return None + return "chat", [ + {"role": "user", "content": f"Repository: {row['repo']}\n\n{issue}\n\nWrite the patch."}, + {"role": "assistant", "content": row["patch"]}, + ] + + +def adapt_advisory(row: dict[str, Any]) -> tuple[str, Any] | None: + if not row.get("description"): + return None + head = row["id"] + (f": {row['title']}" if row.get("title") else "") + return "doc", f"{head}\n{row['description']}" + + +def adapt_stackexchange(row: dict[str, Any]) -> tuple[str, Any] | None: + return "doc", row["text"] + + +def adapt_business_pdf(row: dict[str, Any]) -> tuple[str, Any] | None: + ocr = row.get("ocr_quality_scores") or [0] + if min(ocr) < 2.0 or not BUSINESS_WORDS.search(row["text"][:4000]): + return None + return "doc", row["text"] + + +def adapt_email(row: dict[str, Any]) -> tuple[str, Any] | None: + if len(row.get("body") or "") < 200: + return None + return "doc", f"From: {row['from']}\nTo: {', '.join(row.get('to') or [])}\nSubject: {row.get('subject', '')}\n\n{row['body']}" + + +def adapt_wildchat(row: dict[str, Any]) -> tuple[str, Any] | None: + # Human prompts only: the responses are OpenAI-model output, so they stay out. + if row.get("language") != "English" or row.get("toxic") or row.get("redacted"): + return None + first = next((m["content"] for m in row["conversation"] if m["role"] == "user"), "") + return ("chat", [{"role": "user", "content": first}]) if len(first) > 40 else None + + +def adapt_text(row: dict[str, Any]) -> tuple[str, Any] | None: + return "doc", row["text"] + + +ADAPTERS: dict[str, Callable[[dict[str, Any]], tuple[str, Any] | None]] = { + "openai": adapt_openai, "terminus": adapt_terminus, "toucan": adapt_toucan, "issue_patch": adapt_issue_patch, + "advisory": adapt_advisory, "stackexchange": adapt_stackexchange, + "business_pdf": adapt_business_pdf, "email": adapt_email, "wildchat": adapt_wildchat, "text": adapt_text, +} # fmt: skip + + +def rfc_rows(first: int, last: int) -> Iterator[dict[str, Any]]: + for number in range(first, last + 1): + try: + with urllib.request.urlopen(f"https://www.rfc-editor.org/rfc/rfc{number}.txt", timeout=30) as response: + yield {"rfc": number, "text": response.read().decode("utf-8", "replace")} + except OSError: + continue # unassigned or not-yet-published numbers + + +# --------------------------------------------------------------------------- +# Rendering, splitting, packing +# --------------------------------------------------------------------------- + + +class Renderer: + def __init__(self, model_dir: Path): + from tokenizers import Tokenizer + + sys.path.insert(0, str(model_dir / "encoding")) + import encoding # V4.1's own chat format (the pinned revision's encoding/encoding.py) + + self.encoding = encoding + self.tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json")) + self.bos = self.tokenizer.token_to_id(encoding.bos_token) + self.eos = self.tokenizer.token_to_id(encoding.eos_token) + + def __call__(self, kind: str, payload: Any) -> list[int]: + if kind == "chat": + thinking = any(m.get("reasoning_content") for m in payload) + text = self.encoding.encode_messages(payload, thinking_mode="thinking" if thinking else "chat") + return self.tokenizer.encode(text, add_special_tokens=False).ids + return [self.bos, *self.tokenizer.encode(payload, add_special_tokens=False).ids] + + +def is_eval_group(key: str) -> bool: + return int(hashlib.sha256(key.encode()).hexdigest(), 16) % 10 == 0 + + +def iter_rows(source: dict[str, Any], revision: str | None) -> Iterator[dict[str, Any]]: + if source["adapter"] == "rfc": + yield from rfc_rows(**source["load"]) + return + from datasets import load_dataset + + load = dict(source["load"]) + yield from load_dataset(source["dataset"], streaming=True, revision=revision, **load) + + +def build(sources: list[dict[str, Any]], render: Callable[[str, Any], list[int]], *, eos: int, max_tokens: int, + pack_tokens: int, session_tokens: int, revisions: dict[str, str | None], rows: Callable = iter_rows) -> tuple[list[dict], dict]: # fmt: skip + records: list[dict[str, Any]] = [] + report: dict[str, Any] = {"sources": {}, "tokens": defaultdict(lambda: Counter())} + for source in sources: + adapter = ADAPTERS.get(source["adapter"]) + need = {d: {"calibration": c, "eval": e} for d, (c, e) in source["budget"].items()} + got = report["tokens"] + stats = Counter() + buffers: dict[tuple[str, str], list[int]] = {} + groups: dict[tuple[str, str], list[str]] = defaultdict(list) + + def emit(domain: str, split: str, ids: list[int], keys: list[str]) -> None: + records.append({"id": f"{source['name']}-{len(records)}", "split": split, "domain": domain, + "source": source["name"], "groups": keys, "token_ids": ids[:max_tokens]}) # fmt: skip + got[domain][split] += min(len(ids), max_tokens) + + for row in rows(source, revisions.get(source["name"])): + if all(got[d][s] >= n for d, v in need.items() for s, n in v.items()): + break + stats["rows"] += 1 + domain = source["domain"](row) if callable(source["domain"]) else source["domain"] + if domain not in need: + stats["off_domain"] += 1 + continue + converted = adapt_text({"text": row["text"]}) if source["adapter"] == "rfc" else adapter(row) + if converted is None: + stats["filtered"] += 1 + continue + if not is_english(json.dumps(converted[1], ensure_ascii=False)): + stats["non_english"] += 1 + continue + key = str(row.get(source["group"])) + if source.get("group_host"): # hold out whole sites, not single URLs + key = urlparse(key).netloc or key + split = "eval" if is_eval_group(key) else "calibration" + if got[domain][split] >= need[domain][split]: + stats["over_budget"] += 1 + continue + ids = render(*converted) + stats["kept"] += 1 + if not source.get("pack"): + emit(domain, split, ids, [key]) + continue + buf = buffers.setdefault((domain, split), []) + buf.extend(ids + [eos]) + groups[(domain, split)].append(key) + if len(buf) >= pack_tokens: + emit(domain, split, buf, groups.pop((domain, split))) + buffers[(domain, split)] = [] + for (domain, split), buf in buffers.items(): + if buf: + emit(domain, split, buf, groups[(domain, split)]) + report["sources"][source["name"]] = {"dataset": source["dataset"], "revision": revisions.get(source["name"]), **stats} + + # Sessions: each eval record alone; calibration interleaved across domains, cut at session_tokens. + calibration = sorted((r for r in records if r["split"] == "calibration"), key=lambda r: hashlib.sha256(r["id"].encode()).hexdigest()) + session, size = 0, 0 + for record in calibration: + if size and size + len(record["token_ids"]) > session_tokens: + session, size = session + 1, 0 + record["session"] = f"cal-{session:04d}" + record["target_spans"] = [] + size += len(record["token_ids"]) + for i, record in enumerate(r for r in records if r["split"] == "eval"): + record["session"] = f"eval-{i:05d}" + record["target_spans"] = [[1, len(record["token_ids"])]] + report["tokens"] = {d: dict(c) for d, c in sorted(report["tokens"].items())} + return calibration + [r for r in records if r["split"] == "eval"], report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--model-dir", required=True, type=Path, help="V4.1 HF snapshot (tokenizer.json, encoding/)") + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--max-tokens", type=int, default=16384) + parser.add_argument("--pack-tokens", type=int, default=6144) + parser.add_argument("--session-tokens", type=int, default=131072) + parser.add_argument("--only", help="comma-separated source names (debugging)") + args = parser.parse_args(argv) + + from huggingface_hub import HfApi + + sources = [s for s in SOURCES if not args.only or s["name"] in args.only.split(",")] + api = HfApi() + revisions = {s["name"]: (api.dataset_info(s["dataset"]).sha if "/" in s["dataset"] else None) for s in sources} + render = Renderer(args.model_dir) + records, report = build(sources, render, eos=render.eos, max_tokens=args.max_tokens, pack_tokens=args.pack_tokens, + session_tokens=args.session_tokens, revisions=revisions) # fmt: skip + args.out.mkdir(parents=True, exist_ok=True) + pack = args.out / "v41-pack.jsonl" + with pack.open("w") as handle: + for record in records: + handle.write(json.dumps(record) + "\n") + report.update( + record_type="keep_v41_reap_pack_v1", + pack_sha256=hashlib.sha256(pack.read_bytes()).hexdigest(), + tokenizer_sha256=hashlib.sha256((args.model_dir / "tokenizer.json").read_bytes()).hexdigest(), + encoding_sha256=hashlib.sha256((args.model_dir / "encoding" / "encoding.py").read_bytes()).hexdigest(), + records=len(records), + sessions=len({r["session"] for r in records}), + settings={"max_tokens": args.max_tokens, "pack_tokens": args.pack_tokens, "session_tokens": args.session_tokens}, + ) + (args.out / "manifest.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({k: report[k] for k in ("records", "sessions", "tokens")}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/runpod/v41_reap/collect.py b/scripts/runpod/v41_reap/collect.py new file mode 100644 index 0000000..41e1e00 --- /dev/null +++ b/scripts/runpod/v41_reap/collect.py @@ -0,0 +1,552 @@ +"""Teacher pass for DeepSeek-V4.1-Flash on a GPU node: REAP, imatrix, routes, logits. + +Runs DeepSeek's own reference inference code (``inference/model.py`` at the +pinned revision) under ``torchrun``, with one change: ``MoE.forward`` is +replaced by a copy that also records, for every routed slot, + +- REAP statistics (``keep.quality.reap.ReapStats`` arrays, ``reap__*``); +- imatrix sums for the two expert input spaces (``hidden`` = the post + ``ffn_norm`` row that w1/w3 read, ``down`` = the GLU activation w2 reads), + under the keys ``keep.quality.dsv4_teacher_runner`` finalizes; +- the first ``ROUTE_RECORD_TOKENS`` routes of each session (``routes__*``), + for ``benchmarks/replay_dsv4_expert_residency.py``. + +Eval sessions also capture top-K teacher logits at their supervised positions, +in the teacher runner's ``logits`` schema. Output, per session, is one npz under +``//sessions/``, so KEEP's existing ``finalize-reap`` and imatrix +finalizers read it unchanged. + +The replacement forward computes exactly what the reference does (same ops, +same order); ``tests/test_runpod_v41_reap.py`` checks it, bit for bit, +against a CPU copy of the reference Gate / Expert / MoE. One approximation: the reference folds the gate weight +in *before* w2 (``w2(g * h)``), so the expert output norm ``||f||`` is taken as +``||w2(g * h)|| / g``. With FP8 activation quantization in w2 that differs +from ``||w2(h)||`` by rounding only. + +Usage (on the pod, from the reference ``inference`` directory):: + + torchrun --nproc-per-node 8 /workspace/keep/scripts/runpod/v41_reap/collect.py \\ + --ckpt-path /workspace/v41-tp8 --config config.json \\ + --pack /workspace/pack/v41-pack.jsonl --out /workspace/out \\ + --deadline-utc 2026-11-07T06:00:00Z + +Then, after finalize-reap and reap-select on the teacher pass, the same command +with ``--survivors 192=/workspace/out/survivors-192.json`` (repeatable) scores +each candidate expert count against the teacher in ``/student-KEEP/``. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import sys +import time +from collections import OrderedDict +from pathlib import Path +from typing import Any + +import numpy as np + +MODEL_ID = "deepseek-ai/DeepSeek-V4.1-Flash-1017" +PINNED_REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170" +#: sha256 of inference/model.py at PINNED_REVISION. The patched forward below +#: mirrors that file's MoE.forward and Expert.forward; refuse anything else. +REFERENCE_MODEL_PY_SHA256 = "e5decd76fe70fd56ef2f10124d245c6f64fb6a0003cc9068a1d3182a168b8d2d" +ROUTE_RECORD_TOKENS = 2048 # matches keep.quality.expert_residency +REAP_PREFIX = "reap__" +ROUTE_PREFIX = "routes__" +SPACES = ("hidden", "down") +CAPTURE_KEYS = ( + "topk_logit_ids", "topk_logit_values", "logsumexp", "tail_mass", "target_logit", # teacher pass + "kl_on_teacher_topk", "student_nll", "top1_agree", # masked-router student pass +) + + +# --------------------------------------------------------------------------- +# Accumulation (torch, float64 on device; one instance per process) +# --------------------------------------------------------------------------- + + +class SessionStats: + """Per-session sums over this rank's local experts; all-reduced at the end.""" + + def __init__(self, *, torch: Any, n_layers: int, n_experts: int, dims: dict[str, int], device: Any): + self.torch = torch + self.n_layers, self.n_experts = n_layers, n_experts + f64 = dict(dtype=torch.float64, device=device) + z = lambda *shape: torch.zeros(shape, **f64) + self.reap = { + "route_count": z(n_layers, n_experts), + "gate_sum": z(n_layers, n_experts), + "gate_norm_sum": z(n_layers, n_experts), + "norm_sum": z(n_layers, n_experts), + "norm_max": z(n_layers, n_experts), + "abs_max": z(n_layers, n_experts), + } + self.routed_slots = z(n_layers) + self.imatrix = { + space: { + "importance_sum": z(n_layers, n_experts, dim), + "affinity_weighted_importance": z(n_layers, n_experts, dim), + "route_count": z(n_layers, n_experts), + "affinity_score_sum": z(n_layers, n_experts), + "total_route_count": z(n_layers), + } + for space, dim in dims.items() + } + self.route_ids: list[list[np.ndarray]] = [[] for _ in range(n_layers)] + self.route_gates: list[list[np.ndarray]] = [[] for _ in range(n_layers)] + self.route_tokens = [0] * n_layers + + def add_routes(self, layer: int, indices: Any, weights: Any, *, record: bool) -> None: + """Once per layer per forward, on every rank (the gate is replicated).""" + + rows, top_k = indices.shape + if record: # rank 0 only: identical on every rank + self.routed_slots[layer] += rows * top_k + for space in SPACES: + self.imatrix[space]["total_route_count"][layer] += rows * top_k + room = ROUTE_RECORD_TOKENS - self.route_tokens[layer] + if room > 0: + self.route_ids[layer].append(indices[:room].to("cpu").numpy().astype(np.int16)) + self.route_gates[layer].append(weights[:room].float().to("cpu").numpy().astype(np.float16)) + self.route_tokens[layer] += min(room, rows) + + def add_expert(self, layer: int, expert: int, x: Any, gate: Any, out: Any, h: Any) -> None: + """One local expert's routed rows: input ``x``, gate ``gate`` [r], output, GLU ``h``.""" + + torch = self.torch + g = gate.double() + out64 = out.double() + weighted_norm = out64.norm(dim=-1) # ||g f|| (g > 0) + f_norm = weighted_norm / g + r = self.reap + r["route_count"][layer, expert] += g.numel() + r["gate_sum"][layer, expert] += g.sum() + r["gate_norm_sum"][layer, expert] += weighted_norm.sum() + r["norm_sum"][layer, expert] += f_norm.sum() + r["norm_max"][layer, expert] = torch.maximum(r["norm_max"][layer, expert], f_norm.max()) + abs_max = (out64.abs().amax(dim=-1) / g).max() + r["abs_max"][layer, expert] = torch.maximum(r["abs_max"][layer, expert], abs_max) + for space, rows in (("hidden", x), ("down", h)): + sq = rows.double().square() + m = self.imatrix[space] + m["importance_sum"][layer, expert] += sq.sum(dim=0) + m["affinity_weighted_importance"][layer, expert] += (sq * g[:, None]).sum(dim=0) + m["route_count"][layer, expert] += g.numel() + m["affinity_score_sum"][layer, expert] += g.sum() + + def reduce(self, dist: Any) -> None: + if dist is None or not dist.is_initialized(): + return + for name, value in self.reap.items(): + op = dist.ReduceOp.MAX if name in ("norm_max", "abs_max") else dist.ReduceOp.SUM + dist.all_reduce(value, op=op) + dist.all_reduce(self.routed_slots) + for space in SPACES: + for value in self.imatrix[space].values(): + dist.all_reduce(value) + + def arrays(self, layers: list[int]) -> dict[str, np.ndarray]: + np_ = lambda t: t.to("cpu").numpy() + out: dict[str, np.ndarray] = { + "layers": np.asarray(layers, dtype=np.int32), + "num_experts": np.asarray(self.n_experts, dtype=np.int32), + f"{REAP_PREFIX}layers": np.asarray(layers, dtype=np.int64), + f"{REAP_PREFIX}num_experts": np.asarray(self.n_experts, dtype=np.int64), + f"{REAP_PREFIX}routed_slots": np.rint(np_(self.routed_slots)).astype(np.int64), + } + for name, value in self.reap.items(): + arr = np_(value) + out[f"{REAP_PREFIX}{name}"] = np.rint(arr).astype(np.int64) if name == "route_count" else arr + for space in SPACES: + m = self.imatrix[space] + # float32 on disk, as KEEP's validator requires (sums stay float64 until here) + out[f"importance_sum__{space}"] = np_(m["importance_sum"]).astype(np.float32) + out[f"affinity_weighted_importance__{space}"] = np_(m["affinity_weighted_importance"]).astype(np.float32) + out[f"route_count__{space}"] = np.rint(np_(m["route_count"])).astype(np.int64) + out[f"affinity_score_sum__{space}"] = np_(m["affinity_score_sum"]) + out[f"total_route_count__{space}"] = np.rint(np_(m["total_route_count"])).astype(np.int64) + if all(self.route_ids[layer] for layer in range(self.n_layers)): + n = min(self.route_tokens) + out[f"{ROUTE_PREFIX}layers"] = np.asarray(layers, dtype=np.int64) + out[f"{ROUTE_PREFIX}expert_ids"] = np.stack( + [np.concatenate(self.route_ids[layer])[:n] for layer in range(self.n_layers)], axis=1 + ) + out[f"{ROUTE_PREFIX}gate_weights"] = np.stack( + [np.concatenate(self.route_gates[layer])[:n] for layer in range(self.n_layers)], axis=1 + ) + return out + + +# --------------------------------------------------------------------------- +# Patched reference modules +# --------------------------------------------------------------------------- + + +def expert_forward_with_activation(F: Any, torch: Any, expert: Any, x: Any, weights: Any) -> tuple[Any, Any]: + """``Expert.forward(x, weights)`` from the reference, also returning the GLU activation.""" + + dtype = x.dtype + gate = expert.w1(x).float() + up = expert.w3(x).float() + if expert.swiglu_limit > 0: + up = torch.clamp(up, min=-expert.swiglu_limit, max=expert.swiglu_limit) + gate = torch.clamp(gate, max=expert.swiglu_limit) + h = F.silu(gate) * up + return expert.w2((weights * h).to(dtype)), h + + +def make_moe_forward(ref: Any, torch: Any, F: Any, dist: Any, state: dict[str, Any]): + """A drop-in ``MoE.forward`` that records into ``state['stats']`` when set.""" + + def forward(self, x, image_mask=None): + shape = x.size() + x = x.view(-1, self.dim) + weights, indices = self.gate(x, None if image_mask is None else image_mask.flatten()) + y = torch.zeros_like(x, dtype=torch.float32) + counts = torch.bincount(indices.flatten(), minlength=self.n_routed_experts).tolist() + stats = state.get("stats") + layer = self.layer_id if stats is not None and self.layer_id < stats.n_layers else None + if layer is not None: + stats.add_routes(layer, indices, weights, record=state["rank"] == 0) + for i in range(self.experts_start_idx, self.experts_end_idx): + if counts[i] == 0: + continue + expert = self.experts[i] + idx, top = torch.where(indices == i) + w = weights[idx, top, None] + if layer is None: + y[idx] += expert(x[idx], w) + continue + out, h = expert_forward_with_activation(F, torch, expert, x[idx], w) + stats.add_expert(layer, i, x[idx], w[:, 0], out, h) + y[idx] += out + if ref.world_size > 1: + dist.all_reduce(y) + y += self.shared_experts(x) + return y.type_as(x).view(shape) + + return forward + + +def make_head_forward(head: Any, torch: Any, F: Any, dist: Any, state: dict[str, Any], *, top_k: int, rows_per_slice: int): + """``ParallelHead.forward`` that also captures top-K logits at ``state['positions']``.""" + + original = head.forward + + def forward(x, full_logits=False): + positions = state.get("positions") + if positions is not None: + picked = x[0, positions] # [P, dim], already normalized + for start in range(0, picked.shape[0], rows_per_slice): + part = F.linear(picked[start : start + rows_per_slice].float(), head.weight) + if dist is not None and dist.is_initialized() and dist.get_world_size() > 1: + gathered = [torch.empty_like(part) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, part) + part = torch.cat(gathered, dim=-1) + lse = torch.logsumexp(part, dim=-1) + teacher = state.get("teacher") + if teacher is not None: # masked-router student pass: score against the teacher, keep scalars + rows = slice(start, start + part.shape[0]) + t_logp = teacher["logprob"][rows] + s_logp = part.gather(1, teacher["ids"][rows]) - lse[:, None] + kl = (t_logp.exp() * (t_logp - s_logp)).sum(dim=-1) + nll = lse - part.gather(1, teacher["targets"][rows, None])[:, 0] + agree = part.argmax(dim=-1) == teacher["ids"][rows, 0] + if state["rank"] == 0: + cap = state["captured"] + cap["kl_on_teacher_topk"].append(kl.cpu().numpy().astype(np.float32)) + cap["student_nll"].append(nll.cpu().numpy().astype(np.float32)) + cap["top1_agree"].append(agree.cpu().numpy()) + continue + values, ids = part.topk(top_k, dim=-1) + tail = (1.0 - torch.exp(values - lse[:, None]).sum(dim=-1)).clamp_min(0.0) + target = part.gather(1, state["targets"][start : start + part.shape[0], None])[:, 0] + if state["rank"] == 0: + cap = state["captured"] + cap["target_logit"].append(target.cpu().numpy().astype(np.float32)) + cap["topk_logit_ids"].append(ids.to(torch.int32).cpu().numpy()) + cap["topk_logit_values"].append(values.cpu().numpy().astype(np.float16)) + cap["logsumexp"].append(lse.cpu().numpy().astype(np.float32)) + cap["tail_mass"].append(tail.cpu().numpy().astype(np.float16)) + return original(x, full_logits) + + return forward + + +# --------------------------------------------------------------------------- +# Pack and output +# --------------------------------------------------------------------------- + + +def load_sessions(pack: Path) -> OrderedDict[str, list[dict[str, Any]]]: + """Pack lines grouped by ``session`` in file order. + + Each line: ``{"id", "session", "split": "calibration"|"eval", "domain", + "token_ids": [...], "target_spans": [[start, end], ...]}``. Target spans + index ``token_ids``; logits are captured at ``span_index - 1``. + """ + + sessions: OrderedDict[str, list[dict[str, Any]]] = OrderedDict() + with pack.open() as handle: + for line in handle: + if line.strip(): + record = json.loads(line) + sessions.setdefault(record["session"], []).append(record) + for name, records in sessions.items(): + splits = {r["split"] for r in records} + if len(splits) != 1 or not splits <= {"calibration", "eval"}: + raise ValueError(f"session {name} mixes or misnames splits: {sorted(splits)}") + if splits == {"eval"} and len(records) != 1: + raise ValueError(f"eval session {name} must hold exactly one sample (one logits stream)") + return sessions + + +def target_positions(record: dict[str, Any]) -> np.ndarray: + n = len(record["token_ids"]) + targets = sorted({t for start, end in record["target_spans"] for t in range(max(int(start), 1), min(int(end), n))}) + return np.asarray(targets, dtype=np.int64) - 1 + + +def sha256_ids(ids: list[int]) -> str: + return hashlib.sha256(np.asarray(ids, dtype=np.int32).tobytes()).hexdigest() + + +def identity(*, session: str, split: str, records: list[dict[str, Any]], config_sha: str, chunk: int) -> dict[str, np.ndarray]: + ids_hash = hashlib.sha256("".join(sha256_ids(r["token_ids"]) for r in records).encode()).hexdigest() + mode = "calibration" if split == "calibration" else "logits" + return { + "record_type": np.asarray(f"dsv4_teacher_{mode}_v1"), + "schema_version": np.asarray(1, dtype=np.int32), + "prompt_id": np.asarray(session), + "campaign_split": np.asarray(split), + "token_ids_sha256": np.asarray(ids_hash), + "token_count": np.asarray(sum(len(r["token_ids"]) for r in records), dtype=np.int64), + "prefill_chunk_tokens": np.asarray(chunk, dtype=np.int32), + "generation_config_sha256": np.asarray(config_sha), + "producer": np.asarray("keep/scripts/runpod/v41_reap/collect.py (DeepSeek reference torch)"), + "model_id": np.asarray(MODEL_ID), + "model_revision": np.asarray(PINNED_REVISION), + "domains": np.asarray(sorted({r["domain"] for r in records})), + "sample_ids": np.asarray([r["id"] for r in records]), + } + + +def save_npz(path: Path, arrays: dict[str, np.ndarray]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp.npz") + np.savez(tmp, **arrays) + tmp.replace(path) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--ckpt-path", required=True, help="convert.py output: model{rank}-mp{N}.safetensors + tokenizer") + parser.add_argument("--config", default="config.json", help="reference inference/config.json") + parser.add_argument("--pack", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--max-seq-len", type=int, default=16384) + parser.add_argument("--top-k", type=int, default=2048, help="teacher logits kept per eval position") + parser.add_argument("--rows-per-slice", type=int, default=256) + parser.add_argument("--deadline-utc", help="stop cleanly after the session running at this time") + parser.add_argument( + "--survivors", action="append", default=[], metavar="KEEP=PATH", + help="student pass (repeatable): mask the router to a reap-select table, score eval sessions " + "against the teacher run in --out, write /student-KEEP/", + ) + parser.add_argument("--skip-hash-check", action="store_true", help="tests only") + args = parser.parse_args(argv) + + import torch + import torch.distributed as dist + import torch.nn.functional as F + from safetensors.torch import load_model + from transformers import AutoTokenizer + + here = Path.cwd() + model_py = here / "model.py" + digest = hashlib.sha256(model_py.read_bytes()).hexdigest() + if not args.skip_hash_check and digest != REFERENCE_MODEL_PY_SHA256: + raise SystemExit(f"{model_py} sha256 {digest} is not the pinned reference ({REFERENCE_MODEL_PY_SHA256})") + sys.path.insert(0, str(here)) + import model as ref + + world_size = int(os.getenv("WORLD_SIZE", "1")) + rank = int(os.getenv("RANK", "0")) + local_rank = int(os.getenv("LOCAL_RANK", "0")) + if world_size > 1: + dist.init_process_group("nccl") + torch.cuda.set_device(local_rank) + torch.cuda.memory._set_allocator_settings("expandable_segments:True") + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(33377335) + + with open(args.config) as f: + config = json.load(f) + model_args = ref.ModelArgs(**config) + model_args.max_batch_size = 1 + model_args.max_seq_len = args.max_seq_len + tokenizer = AutoTokenizer.from_pretrained(args.ckpt_path) + with torch.device("cuda"): + model = ref.Transformer(model_args, tokenizer) + load_model(model, os.path.join(args.ckpt_path, f"model{rank}-mp{world_size}.safetensors")) + torch.set_default_device("cuda") + model.eval() + original_bias = { + (layer, leaf): getattr(block.ffn.gate, leaf).data.clone() + for layer, block in enumerate(model.layers) + for leaf in ("bias", "bias_vl") + if getattr(block.ffn.gate, leaf) is not None + } + + def mask_router(table_path: Path | None) -> str | None: + """Masking the router to -inf equals slicing it (tests/test_reap.py), so one + loaded checkpoint scores every candidate expert count.""" + + for (layer, leaf), value in original_bias.items(): + getattr(model.layers[layer].ffn.gate, leaf).data.copy_(value) + if table_path is None: + return None + for layer, kept in json.loads(table_path.read_text())["survivors"].items(): + gate = model.layers[int(layer)].ffn.gate + mask = torch.ones(gate.bias.shape[0], dtype=torch.bool, device=gate.bias.device) + mask[torch.as_tensor(kept, device=mask.device)] = False + for leaf in ("bias", "bias_vl"): + if (int(layer), leaf) in original_bias: + getattr(gate, leaf).data[mask] = float("-inf") + return hashlib.sha256(table_path.read_bytes()).hexdigest() + + state: dict[str, Any] = {"rank": rank, "stats": None, "positions": None} + ref.MoE.forward = make_moe_forward(ref, torch, F, dist, state) + model.head.forward = make_head_forward(model.head, torch, F, dist, state, top_k=args.top_k, rows_per_slice=args.rows_per_slice) + + run_config = { + "model_id": MODEL_ID, + "revision": PINNED_REVISION, + "model_py_sha256": digest, + "pack_sha256": hashlib.sha256(args.pack.read_bytes()).hexdigest(), + "max_seq_len": args.max_seq_len, + "top_k": args.top_k, + "world_size": world_size, + } + config_sha = hashlib.sha256(json.dumps(run_config, sort_keys=True).encode()).hexdigest() + if rank == 0: + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "run-config.json").write_text(json.dumps(run_config, indent=2, sort_keys=True) + "\n") + + deadline = None + if args.deadline_utc: + deadline = dt.datetime.fromisoformat(args.deadline_utc.replace("Z", "+00:00")).timestamp() + n_layers = int(model_args.n_layers) + n_experts = int(model_args.n_routed_experts) + dims = {"hidden": int(model_args.dim), "down": int(model_args.moe_inter_dim)} + layers = list(range(n_layers)) + sessions = load_sessions(args.pack) + started = time.time() + teacher_config_sha = hashlib.sha256((args.out / "run-config.json").read_bytes()).hexdigest() if rank == 0 else "" + + def run_pass(out_dir: Path, survivors_sha: str | None) -> bool: + """One pass over the pack. Teacher pass when ``survivors_sha`` is None, else a + masked-router student pass over the eval sessions. False once the deadline hits.""" + + student = survivors_sha is not None + progress = out_dir / "progress.jsonl" + for name, records in sessions.items(): + split = records[0]["split"] + target = out_dir / split / "sessions" / f"{name}.npz" + if target.exists() or (student and split != "eval"): + continue + if deadline is not None and time.time() > deadline: + if rank == 0: + print(f"deadline reached before session {name}; stopping cleanly", flush=True) + return False + t0 = time.time() + stats = None if student else SessionStats(torch=torch, n_layers=n_layers, n_experts=n_experts, dims=dims, device="cuda") + state["stats"] = stats + state["captured"] = {k: [] for k in CAPTURE_KEYS} + if student: + with np.load(args.out / "eval" / "sessions" / f"{name}.npz") as teacher: + state["teacher"] = { + "ids": torch.as_tensor(teacher["topk_logit_ids"].astype(np.int64), device="cuda"), + "logprob": torch.as_tensor( + teacher["topk_logit_values"].astype(np.float32) - teacher["logsumexp"][:, None], device="cuda" + ), + "targets": torch.as_tensor(teacher["target_token_ids"].astype(np.int64), device="cuda"), + } + tokens_seen = 0 + logits_arrays: dict[str, np.ndarray] = {} + for record in records: + ids = record["token_ids"] + if len(ids) > args.max_seq_len: + raise ValueError(f"{record['id']} has {len(ids)} tokens > --max-seq-len") + positions = target_positions(record) if split == "eval" else None + state["positions"] = None if positions is None else torch.as_tensor(positions, device="cuda") + if positions is not None: + state["targets"] = torch.as_tensor(np.asarray(ids, dtype=np.int64)[positions + 1], device="cuda") + with torch.inference_mode(): + model.forward(torch.as_tensor([ids], dtype=torch.long, device="cuda"), 0) + tokens_seen += len(ids) + if positions is not None: + logits_arrays = { + "positions": positions.astype(np.int32), + "target_token_ids": np.asarray(ids, dtype=np.int32)[positions + 1], + } + if rank == 0: + logits_arrays.update({k: np.concatenate(v) for k, v in state["captured"].items() if v}) + state["positions"] = state["targets"] = state["teacher"] = state["stats"] = None + if stats is not None: + stats.reduce(dist) + if rank == 0: + arrays = identity(session=name, split=split, records=records, config_sha=config_sha, chunk=args.max_seq_len) + if stats is not None: + stat_arrays = stats.arrays(layers) + if split == "eval": # REAP + routes only: per-split saliency, not ~0.9 GB of imatrix per sample + stat_arrays = {k: v for k, v in stat_arrays.items() if k.split("__")[-1] not in SPACES} + arrays.update(stat_arrays) + arrays.update(logits_arrays) + if student: + arrays["record_type"] = np.asarray("dsv4_masked_student_eval_v1") + arrays["survivors_sha256"] = np.asarray(survivors_sha) + arrays["teacher_run_config_sha256"] = np.asarray(teacher_config_sha) + save_npz(target, arrays) + entry = { + "session": name, + "split": split, + "tokens": tokens_seen, + "seconds": round(time.time() - t0, 3), + "tok_per_s": round(tokens_seen / max(time.time() - t0, 1e-9), 1), + "elapsed_h": round((time.time() - started) / 3600, 3), + } + with progress.open("a") as handle: + handle.write(json.dumps(entry) + "\n") + print(json.dumps(entry), flush=True) + del stats + torch.cuda.empty_cache() + return True + + passes: list[tuple[Path, Path | None]] = [(args.out, None)] + for item in args.survivors: + keep, _, table = item.partition("=") + passes.append((args.out / f"student-{int(keep)}", Path(table))) + for out_dir, table in passes: + if rank == 0: + print(f"pass -> {out_dir}", flush=True) + if not run_pass(out_dir, mask_router(table)): + break + if world_size > 1: + dist.barrier() + dist.destroy_process_group() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/pod.sh b/scripts/runpod/v41_reap/pod.sh new file mode 100755 index 0000000..5ec4d09 --- /dev/null +++ b/scripts/runpod/v41_reap/pod.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# DeepSeek-V4.1-Flash REAP run on RunPod. Runbook: +# docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md +# +# pod.sh fetch CPU pod: download the pinned checkpoint, verify every LFS sha256 +# pod.sh pack CPU pod: build the calibration + eval pack from public datasets +# pod.sh gpu 8xH200 pod: convert, teacher pass, candidate scoring; stops itself +# pod.sh prune KEEP CPU pod: write the pruned HF checkpoint for the Mac to pull +# +# Everything lives on one network volume mounted at /workspace, so stages can +# run on different (cheap CPU / expensive GPU) pods. +set -euo pipefail + +MODEL_ID=deepseek-ai/DeepSeek-V4.1-Flash-1017 +REVISION=1bae0e1088993b9b82de08f59de5be5a34e42170 +W=${WORKSPACE:-/workspace} +KIT=$W/keep/scripts/runpod/v41_reap +HF=$W/v41-hf +TP8=$W/v41-tp8 +OUT=$W/out +CANDIDATES=${CANDIDATES:-"256 224 192 160"} +MAX_GPU_HOURS=${MAX_GPU_HOURS:-6} # hard stop for the GPU pod (8xH200 secure ~ $42/h) +export PYTHONPATH=$W/keep/src HF_HUB_ENABLE_HF_TRANSFER=1 + +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$W/pod.log"; } + +stop_pod() { + log "stopping pod: $*" + if [[ -n "${RUNPOD_POD_ID:-}" ]] && command -v runpodctl >/dev/null; then + runpodctl stop pod "$RUNPOD_POD_ID" || log "runpodctl stop FAILED: stop the pod by hand" + else + log "no runpodctl/RUNPOD_POD_ID: stop the pod by hand" + fi +} + +fetch() { + pip install -q "huggingface_hub[hf_transfer]>=0.34" + hf download "$MODEL_ID" --revision "$REVISION" --local-dir "$HF" + python3 - "$HF" "$MODEL_ID" "$REVISION" <<'EOF' +import hashlib, sys +from pathlib import Path +from huggingface_hub import HfApi +root, repo, rev = Path(sys.argv[1]), sys.argv[2], sys.argv[3] +bad = 0 +for f in HfApi().model_info(repo, revision=rev, files_metadata=True).siblings: + path = root / f.rfilename + want = f.lfs.sha256 if f.lfs else None + if not path.exists() or (f.size is not None and path.stat().st_size != f.size): + print("MISSING/SIZE", f.rfilename); bad += 1; continue + if want: + h = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(64 << 20), b""): + h.update(chunk) + if h.hexdigest() != want: + print("SHA256", f.rfilename); bad += 1 +sys.exit(f"{bad} files failed verification" if bad else 0) +EOF + sha256sum "$HF/inference/model.py" + log "fetch verified" +} + +pack() { + pip install -q "datasets>=3" tokenizers huggingface_hub + python3 -I "$KIT/build_pack.py" --model-dir "$HF" --out "$W/pack" + log "pack built: $(python3 -c "import json;m=json.load(open('$W/pack/manifest.json'));print(m['records'],'records',m['sessions'],'sessions')")" +} + +gpu() { + local deadline + deadline=$(date -u -d "+$((MAX_GPU_HOURS * 60 - 20)) minutes" +%FT%TZ) + # Watchdog: whatever happens below, the pod stops at MAX_GPU_HOURS. + ( sleep $((MAX_GPU_HOURS * 3600)); stop_pod "watchdog: ${MAX_GPU_HOURS}h limit" ) & + trap 'stop_pod "gpu stage exited (status $?)"' EXIT + + cd "$HF/inference" + pip install -q -r requirements.txt + pip freeze > "$W/pip-freeze-gpu.txt" + nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$W/pod.log" + + if [[ ! -f "$TP8/model7-mp8.safetensors" ]]; then + log "convert -> $TP8" + python3 convert.py --hf-ckpt-path "$HF" --save-path "$TP8" --model-parallel 8 --expert-dtype fp4 + fi + + log "teacher pass" + torchrun --nproc-per-node 8 "$KIT/collect.py" --ckpt-path "$TP8" --config config.json \ + --pack "$W/pack/v41-pack.jsonl" --out "$OUT" --deadline-utc "$deadline" + + log "finalize-reap + candidates: $CANDIDATES" + python3 "$W/keep/benchmarks/produce_dsv4_teacher_cache.py" finalize-reap --out-dir "$OUT/calibration" + local survivors=() + for keep in $CANDIDATES; do + python3 "$W/keep/benchmarks/produce_dsv4_teacher_cache.py" reap-select \ + --stats "$OUT/calibration/reap-stats.npz" --keep "$keep" --output "$OUT/survivors-$keep.json" \ + > "$OUT/reap-select-$keep.json" + survivors+=(--survivors "$keep=$OUT/survivors-$keep.json") + done + + log "student passes" + torchrun --nproc-per-node 8 "$KIT/collect.py" --ckpt-path "$TP8" --config config.json \ + --pack "$W/pack/v41-pack.jsonl" --out "$OUT" --deadline-utc "$deadline" "${survivors[@]}" + + local students=() + for keep in $CANDIDATES; do students+=(--student "$keep=$OUT/student-$keep"); done + python3 "$KIT/score.py" --teacher "$OUT" "${students[@]}" | tee "$OUT/candidate-scores.json" + log "gpu stage done; scores in $OUT/candidate-scores.json" +} + +prune() { + local keep=${1:?usage: pod.sh prune KEEP} + pip install -q numpy + python3 "$KIT/prune.py" --src "$HF" --survivors "$OUT/survivors-$keep.json" --out "$W/v41-reap$keep" + (cd "$W/v41-reap$keep" && sha256sum -c SHA256SUMS --quiet) && log "pruned checkpoint $W/v41-reap$keep verified" +} + +stage=${1:?usage: pod.sh fetch|pack|gpu|prune KEEP} +shift +"$stage" "$@" diff --git a/scripts/runpod/v41_reap/prune.py b/scripts/runpod/v41_reap/prune.py new file mode 100644 index 0000000..61ca305 --- /dev/null +++ b/scripts/runpod/v41_reap/prune.py @@ -0,0 +1,208 @@ +"""Write a REAP-pruned copy of a DeepSeek-V4.1 HF checkpoint, byte for byte. + +Streams each source shard and writes a shard of the same name holding: + +- every untouched tensor, copied unchanged; +- the kept routed experts, renumbered ``0..keep-1`` in ascending source order; +- the router ``weight`` / ``bias`` / ``bias_vl`` rows of the kept experts. + +Tensors are copied as raw safetensors bytes, so FP8 weights, ue8m0 scales and +packed FP4 experts are never decoded and nothing needs torch. The plan comes +from ``keep.quality.reap.prune_checkpoint_plan``; the drafter (``mtp.*``) is +left untouched unless ``--mtp-survivors`` is given. ``config.json`` and +``inference/config.json`` get the new ``n_routed_experts``; the index and a +sha256 manifest are rewritten. + +Usage:: + + python scripts/runpod/v41_reap/prune.py --src /workspace/v41-hf \\ + --survivors /workspace/out/survivors-192.json --out /workspace/v41-reap192 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import struct +import sys +from pathlib import Path +from typing import Any, BinaryIO + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO / "src")) + +from keep.quality.reap import prune_checkpoint_plan + +INDEX = "model.safetensors.index.json" +COPY_CHUNK = 64 << 20 + + +def read_header(handle: BinaryIO) -> tuple[dict[str, Any], int]: + (size,) = struct.unpack(" None: + src.seek(start) + while length: + chunk = src.read(min(COPY_CHUNK, length)) + if not chunk: + raise OSError("source shard ended early") + dst.write(chunk) + length -= len(chunk) + + +def prune_shard(src_path: Path, dst_path: Path, plan: dict[str, Any]) -> list[str]: + """Write one pruned shard; return the tensor names it holds.""" + + rename, slice_rows, drop = plan["rename"], plan["slice_rows"], set(plan["drop"]) + with src_path.open("rb") as src: + header, base = read_header(src) + metadata = header.pop("__metadata__", None) + jobs: list[tuple[str, dict[str, Any], list[tuple[int, int]]]] = [] + for name in sorted(header, key=lambda n: header[n]["data_offsets"][0]): + info = header[name] + start, end = info["data_offsets"] + if name in drop: + continue + if name in slice_rows: + rows = info["shape"][0] + row_bytes, rem = divmod(end - start, rows) + if rem: + raise ValueError(f"{name}: {end - start} bytes is not {rows} equal rows") + kept = slice_rows[name] + ranges = [(base + start + r * row_bytes, row_bytes) for r in kept] + shape = [len(kept), *info["shape"][1:]] + jobs.append((name, {"dtype": info["dtype"], "shape": shape}, ranges)) + else: + jobs.append((rename.get(name, name), {"dtype": info["dtype"], "shape": info["shape"]}, [(base + start, end - start)])) + out_header: dict[str, Any] = {} if metadata is None else {"__metadata__": metadata} + offset = 0 + for name, info, ranges in jobs: + length = sum(n for _, n in ranges) + out_header[name] = {**info, "data_offsets": [offset, offset + length]} + offset += length + blob = json.dumps(out_header, separators=(",", ":")).encode() + blob += b" " * (-len(blob) % 8) + dst_path.parent.mkdir(parents=True, exist_ok=True) + tmp = dst_path.with_name(dst_path.name + ".tmp") + with tmp.open("wb") as dst: + dst.write(struct.pack(" int: + """Set every ``n_routed_experts`` key (not the drafter's ``dspark_*``); return how many.""" + + hits = 0 + if isinstance(node, dict): + for key, value in node.items(): + if key == "n_routed_experts": + node[key] = keep + hits += 1 + else: + hits += set_expert_count(value, keep) + elif isinstance(node, list): + hits += sum(set_expert_count(item, keep) for item in node) + return hits + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(COPY_CHUNK), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_survivors(path: Path) -> dict[int, list[int]]: + payload = json.loads(path.read_text()) + if payload.get("record_type") != "keep_reap_survivors_v1": + raise ValueError(f"{path} is not a keep_reap_survivors_v1 table") + return {int(block): [int(e) for e in kept] for block, kept in payload["survivors"].items()} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--src", required=True, type=Path, help="HF checkpoint directory") + parser.add_argument("--survivors", required=True, type=Path, help="reap-select output for the backbone layers") + parser.add_argument("--mtp-survivors", type=Path, help="optional survivor table for the mtp.* drafter") + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args(argv) + + index = json.loads((args.src / INDEX).read_text()) + weight_map: dict[str, str] = index["weight_map"] + survivors = load_survivors(args.survivors) + keep = len(next(iter(survivors.values()))) + plan = prune_checkpoint_plan(weight_map, survivors, prefix="layers") + if args.mtp_survivors: + mtp = prune_checkpoint_plan(plan["untouched"], load_survivors(args.mtp_survivors), prefix="mtp") + plan = { + "rename": {**plan["rename"], **mtp["rename"]}, + "slice_rows": {**plan["slice_rows"], **mtp["slice_rows"]}, + "drop": plan["drop"] + mtp["drop"], + } + # ponytail: the drafter's expert count lives under dspark_n_routed_experts; set it by hand if you prune it. + + args.out.mkdir(parents=True, exist_ok=True) + new_map: dict[str, str] = {} + total = 0 + for shard in sorted(set(weight_map.values())): + names = prune_shard(args.src / shard, args.out / shard, plan) + new_map.update({name: shard for name in names}) + total += (args.out / shard).stat().st_size + print(f"{shard}: {len(names)} tensors", flush=True) + + for path in sorted(args.src.rglob("*")): + rel = path.relative_to(args.src) + if path.is_dir() or path.suffix == ".safetensors" or rel.name == INDEX or rel.parts[0].startswith("."): + continue + target = args.out / rel + target.parent.mkdir(parents=True, exist_ok=True) + if rel.name == "config.json": + config = json.loads(path.read_text()) + if set_expert_count(config, keep) == 0: + raise ValueError(f"{rel}: no n_routed_experts key to update") + target.write_text(json.dumps(config, indent=4) + "\n") + else: + shutil.copy2(path, target) + + expected = len(weight_map) - len(plan["drop"]) + if len(new_map) != expected: + raise AssertionError(f"wrote {len(new_map)} tensors, expected {expected}") + out_index = {"metadata": {**index.get("metadata", {}), "total_size": total}, "weight_map": dict(sorted(new_map.items()))} + (args.out / INDEX).write_text(json.dumps(out_index, indent=2) + "\n") + (args.out / "keep-prune.json").write_text( + json.dumps( + { + "record_type": "keep_reap_pruned_checkpoint_v1", + "source_index_sha256": sha256_file(args.src / INDEX), + "survivors_sha256": sha256_file(args.survivors), + "keep": keep, + "tensors_dropped": len(plan["drop"]), + "router_tensors_sliced": len(plan["slice_rows"]), + "experts_renamed": len(plan["rename"]), + }, + indent=2, + ) + + "\n" + ) + manifest = { + str(p.relative_to(args.out)): sha256_file(p) + for p in sorted(args.out.rglob("*")) + if p.is_file() and p.name != "SHA256SUMS" + } + (args.out / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in manifest.items())) + print(json.dumps({"tensors": len(new_map), "bytes": total, "keep": keep})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/score.py b/scripts/runpod/v41_reap/score.py new file mode 100644 index 0000000..7aab6fa --- /dev/null +++ b/scripts/runpod/v41_reap/score.py @@ -0,0 +1,51 @@ +"""Score masked-router student runs against the teacher, per domain. + + python scripts/runpod/v41_reap/score.py --teacher /workspace/out/teacher \\ + --student 256=/workspace/out/student-256 --student 192=/workspace/out/student-192 + +For each candidate: mean KL(teacher || student) over the teacher's top-K, the +NLL increase on the real next token, and top-1 agreement, per domain and +overall (position-weighted). Writes JSON to stdout. +""" + +from __future__ import annotations + +import argparse +import json +from collections import defaultdict +from pathlib import Path + +import numpy as np + + +def score(teacher: Path, student: Path) -> dict[str, dict[str, float]]: + sums: dict[str, np.ndarray] = defaultdict(lambda: np.zeros(4)) + for path in sorted((student / "eval" / "sessions").glob("*.npz")): + with np.load(path) as s, np.load(teacher / "eval" / "sessions" / path.name) as t: + if not np.array_equal(s["positions"], t["positions"]): + raise ValueError(f"{path.name}: positions differ from the teacher's") + teacher_nll = t["logsumexp"] - t["target_logit"] + row = np.array([s["kl_on_teacher_topk"].sum(), (s["student_nll"] - teacher_nll).sum(), s["top1_agree"].sum(), s["positions"].size]) + for domain in [*map(str, s["domains"]), "all"]: + sums[domain] += row + return { + domain: {"kl": kl / n, "nll_delta": dn / n, "top1_agree": agree / n, "positions": int(n)} + for domain, (kl, dn, agree, n) in sorted(sums.items()) + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--teacher", required=True, type=Path) + parser.add_argument("--student", action="append", required=True, help="KEEP=DIR, repeatable") + args = parser.parse_args(argv) + report = {} + for item in args.student: + keep, _, directory = item.partition("=") + report[keep] = score(args.teacher, Path(directory)) + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_runpod_v41_reap.py b/tests/test_runpod_v41_reap.py new file mode 100644 index 0000000..875f902 --- /dev/null +++ b/tests/test_runpod_v41_reap.py @@ -0,0 +1,343 @@ +"""CPU checks for the RunPod V4.1 REAP kit (scripts/runpod/v41_reap). + +The MoE below copies the reference ``inference/model.py`` (pinned revision) +Gate / Expert / MoE with world_size = 1 and nn.Linear in place of the FP8/FP4 +Linear. The patched forward must give the same output and the statistics must +match a direct float64 computation and pass KEEP's own validators. +""" + +from __future__ import annotations + +import importlib.util +import json +import struct +import types +from collections import Counter +from pathlib import Path + +import numpy as np +import pytest + +REPO = Path(__file__).resolve().parents[1] +KIT = REPO / "scripts" / "runpod" / "v41_reap" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(f"v41_{name}", KIT / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- collector + + +def _reference(torch): + nn, F = torch.nn, torch.nn.functional + ref = types.ModuleType("ref") + ref.world_size = 1 + + class Gate(nn.Module): + def __init__(self, n, dim, topk): + super().__init__() + self.topk, self.route_scale = topk, 1.5 + self.weight = nn.Parameter(torch.randn(n, dim) * 0.3) + self.bias = nn.Parameter(torch.randn(n) * 0.1) + + def forward(self, x, image_mask=None): + scores = F.softplus(F.linear(x.float(), self.weight.float())).sqrt() + indices = (scores + self.bias).topk(self.topk, dim=-1)[1] + weights = scores.gather(1, indices) + weights /= weights.sum(dim=-1, keepdim=True) + 1e-20 + weights *= self.route_scale + return weights, indices + + class Expert(nn.Module): + def __init__(self, dim, inter, swiglu_limit=7.0): + super().__init__() + self.w1 = nn.Linear(dim, inter, bias=False) + self.w2 = nn.Linear(inter, dim, bias=False) + self.w3 = nn.Linear(dim, inter, bias=False) + self.swiglu_limit = swiglu_limit + + def forward(self, x, weights=None): + dtype = x.dtype + gate = self.w1(x).float() + up = self.w3(x).float() + if self.swiglu_limit > 0: + up = torch.clamp(up, min=-self.swiglu_limit, max=self.swiglu_limit) + gate = torch.clamp(gate, max=self.swiglu_limit) + x = F.silu(gate) * up + if weights is not None: + x = weights * x + return self.w2(x.to(dtype)) + + class MoE(nn.Module): + def __init__(self, layer_id, dim=16, inter=12, n=8, topk=2): + super().__init__() + self.layer_id, self.dim, self.n_routed_experts = layer_id, dim, n + self.experts_start_idx, self.experts_end_idx = 0, n + self.gate = Gate(n, dim, topk) + self.experts = nn.ModuleList([Expert(dim, inter) for _ in range(n)]) + self.shared_experts = Expert(dim, inter) + + def forward(self, x, image_mask=None): + shape = x.size() + x = x.view(-1, self.dim) + weights, indices = self.gate(x) + y = torch.zeros_like(x, dtype=torch.float32) + counts = torch.bincount(indices.flatten(), minlength=self.n_routed_experts).tolist() + for i in range(self.experts_start_idx, self.experts_end_idx): + if counts[i] == 0: + continue + idx, top = torch.where(indices == i) + y[idx] += self.experts[i](x[idx], weights[idx, top, None]) + y += self.shared_experts(x) + return y.type_as(x).view(shape) + + ref.MoE = MoE + return ref + + +def test_patched_moe_matches_reference_and_direct_stats(): + torch = pytest.importorskip("torch") + F = torch.nn.functional + collect = _load("collect") + torch.manual_seed(0) + torch.set_grad_enabled(False) + ref = _reference(torch) + layers = [ref.MoE(i) for i in range(2)] + [ref.MoE(2)] # layer 2 plays an mtp block + xs = [torch.randn(1, 37, 16) for _ in layers] + expected = [m(x) for m, x in zip(layers, xs)] + + original = ref.MoE.forward + state = {"rank": 0} + stats = collect.SessionStats(torch=torch, n_layers=2, n_experts=8, dims={"hidden": 16, "down": 12}, device="cpu") + state["stats"] = stats + ref.MoE.forward = collect.make_moe_forward(ref, torch, F, None, state) + try: + got = [m(x) for m, x in zip(layers, xs)] + finally: + ref.MoE.forward = original + for a, b in zip(expected, got): + assert torch.equal(a, b) + + arrays = stats.arrays([0, 1]) + for layer, (m, x) in enumerate(zip(layers[:2], xs)): + flat = x.view(-1, 16) + w, idx = m.gate(flat) + count = np.zeros(8) + gnorm = np.zeros(8) + fmax = np.zeros(8) + hid = np.zeros((8, 16)) + down = np.zeros((8, 12)) + for row in range(flat.shape[0]): + for slot in range(idx.shape[1]): + e, g = int(idx[row, slot]), float(w[row, slot]) + ex = m.experts[e] + h = F.silu(ex.w1(flat[row]).clamp(max=7.0)) * ex.w3(flat[row]).clamp(-7.0, 7.0) + out = ex.w2(g * h).double() + count[e] += 1 + gnorm[e] += float(out.norm()) + fmax[e] = max(fmax[e], float(out.norm()) / g) + hid[e] += flat[row].double().square().numpy() + down[e] += h.double().square().detach().numpy() + np.testing.assert_array_equal(arrays["reap__route_count"][layer], count) + np.testing.assert_allclose(arrays["reap__gate_norm_sum"][layer], gnorm, rtol=1e-5) + np.testing.assert_allclose(arrays["reap__norm_max"][layer], fmax, rtol=1e-5) + np.testing.assert_allclose(arrays["importance_sum__hidden"][layer], hid, rtol=1e-5) + np.testing.assert_allclose(arrays["importance_sum__down"][layer], down, rtol=1e-4, atol=1e-6) + assert arrays["routes__expert_ids"].shape == (37, 2, 2) + assert arrays["routes__expert_ids"].dtype == np.int16 + + from keep.quality.dsv4_teacher_runner import validate_dsv4_calibration_capture + from keep.quality.reap import ReapStats + + validate_dsv4_calibration_capture(arrays, layers=[0, 1], num_experts=8) + assert ReapStats.from_arrays(arrays).routed_slots.tolist() == [74, 74] + torch.set_grad_enabled(True) + + +def test_head_capture_matches_full_logits_schema(): + torch = pytest.importorskip("torch") + F = torch.nn.functional + collect = _load("collect") + from keep.quality.dsv4_teacher_runner import validate_dsv4_logits_capture + + head = types.SimpleNamespace(weight=torch.randn(50, 8)) + head.forward = lambda x, full_logits=False: F.linear(x[:, -1].float(), head.weight) + positions = np.array([2, 3, 7]) + targets = torch.as_tensor([5, 9, 11]) + state = {"rank": 0, "positions": torch.as_tensor(positions), "targets": targets, "captured": {k: [] for k in collect.CAPTURE_KEYS}} + head.forward = collect.make_head_forward(head, torch, F, None, state, top_k=10, rows_per_slice=2) + x = torch.randn(1, 9, 8) + head.forward(x) + cap = {k: np.concatenate(v) for k, v in state["captured"].items() if v} + cap["positions"] = positions.astype(np.int32) + cap["target_token_ids"] = np.zeros(3, dtype=np.int32) + validate_dsv4_logits_capture(cap, top_k=10, supervised=3) + full = F.linear(x[0, positions], head.weight) + np.testing.assert_allclose(cap["logsumexp"], torch.logsumexp(full, -1).numpy(), rtol=1e-6) + assert (cap["topk_logit_ids"][:, 0] == full.argmax(-1).numpy()).all() + np.testing.assert_allclose(cap["target_logit"], full.gather(1, targets[:, None])[:, 0].numpy(), rtol=1e-6) + + # Student pass scored against that teacher: the same head gives KL 0 and full agreement. + lse = torch.as_tensor(cap["logsumexp"]) + state["captured"] = {k: [] for k in collect.CAPTURE_KEYS} + state["teacher"] = { + "ids": torch.as_tensor(cap["topk_logit_ids"].astype(np.int64)), + "logprob": torch.as_tensor(cap["topk_logit_values"].astype(np.float32)) - lse[:, None], + "targets": targets, + } + head.forward(x) + student = {k: np.concatenate(v) for k, v in state["captured"].items() if v} + assert set(student) == {"kl_on_teacher_topk", "student_nll", "top1_agree"} + np.testing.assert_allclose(student["kl_on_teacher_topk"], 0.0, atol=2e-3) + assert student["top1_agree"].all() + np.testing.assert_allclose(student["student_nll"], (lse - torch.as_tensor(cap["target_logit"])).numpy(), rtol=1e-5) + + +def test_pack_rules(tmp_path): + collect = _load("collect") + pack = tmp_path / "pack.jsonl" + rows = [ + {"id": "a", "session": "s1", "split": "calibration", "domain": "rust", "token_ids": [1, 2, 3], "target_spans": []}, + {"id": "b", "session": "e1", "split": "eval", "domain": "bash", "token_ids": [1, 2, 3, 4, 5], "target_spans": [[0, 2], [3, 5]]}, + ] + pack.write_text("".join(json.dumps(r) + "\n" for r in rows)) + sessions = collect.load_sessions(pack) + assert list(sessions) == ["s1", "e1"] + # targets 1 and 3, 4 (0 has no predecessor); logits are read one position earlier + assert collect.target_positions(rows[1]).tolist() == [0, 2, 3] + rows.append({**rows[1], "id": "c"}) + pack.write_text("".join(json.dumps(r) + "\n" for r in rows)) + with pytest.raises(ValueError, match="exactly one sample"): + collect.load_sessions(pack) + + +# --------------------------------------------------------------------------- pruner + + +def _write_safetensors(path: Path, tensors: dict[str, tuple[str, list[int], bytes]]) -> None: + header, offset, blobs = {}, 0, [] + for name, (dtype, shape, data) in tensors.items(): + header[name] = {"dtype": dtype, "shape": shape, "data_offsets": [offset, offset + len(data)]} + offset += len(data) + blobs.append(data) + raw = json.dumps(header).encode() + raw += b" " * (-len(raw) % 8) + path.write_bytes(struct.pack(" dict[str, tuple[str, list[int], bytes]]: + data = path.read_bytes() + (size,) = struct.unpack("= 3000 and report["tokens"]["d"]["eval"] >= 400 + sizes = Counter() + for r in cal: + sizes[r["session"]] += len(r["token_ids"]) + assert max(sizes.values()) <= 500 + assert len({r["session"] for r in ev}) == len(ev) and all(r["target_spans"] == [[1, len(r["token_ids"])]] for r in ev) + assert report["sources"]["fake"].get("non_english", 0) == 0 # the CJK row sits after the budgets are met + rows.insert(0, rows.pop()) + _, report = pack.build([source], render, eos=1, max_tokens=64, pack_tokens=0, session_tokens=500, + revisions={}, rows=lambda s, r: iter(rows)) # fmt: skip + assert report["sources"]["fake"]["non_english"] == 1 From 751fa6400412c4a4b5c9599d2bee5a4886caf21a Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:00:58 -0800 Subject: [PATCH 18/74] runpod: single-pod mode (all stage), REAP 50% and 75% prunes 8xH200 was out of capacity in every data center that offers network volumes, so 'pod.sh all' now runs the whole job on one pod: - fetch and pack in parallel, then convert, the teacher pass and scoring (N = 256/192/128/96); - delete the TP8 copy, then prune N=192 (REAP 50%) and N=96 (REAP 75%). The pod stays up for the Mac to pull the results; the watchdog stops it at MAX_GPU_HOURS, and any failed stage stops it at once. --- .../2026-11-05-v41-runpod-reap-runbook.md | 26 ++++++++++- scripts/runpod/v41_reap/pod.sh | 43 +++++++++++++++---- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md index 1ac3e76..19e3af2 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md @@ -103,7 +103,31 @@ reference prefill plus a Python expert loop. Check it on the first sessions. If `tok_per_s` is under 1,000 after the first 3 sessions, stop and rethink. At that rate the teacher pass alone takes more than an hour. -## Steps +## Single-pod mode (what actually ran, 2026-11-06) + +On 2026-11-06, 8×H200 was out of capacity in every data center that offers +network volumes, so everything runs on one 8×H200 pod with a 2 TB volume disk: + +```bash +MAX_GPU_HOURS=8 bash /workspace/keep/scripts/runpod/v41_reap/pod.sh all +``` + +That command runs these steps: + +- **fetch and pack:** run in parallel. +- **convert:** writes the TP8 checkpoint. +- **teacher pass:** runs over the pack. +- **scoring:** checks N = 256/192/128/96. +- **cleanup:** deletes the TP8 checkpoint. +- **prune:** writes REAP 50% (N=192, ~366 GB) and REAP 75% (N=96, ~293 GB). + +The pod stays up so the Mac can pull the results. The watchdog stops it at +8 h, and a failed stage stops it at once. + +The Mac cannot hold both checkpoints at once (423 GB free), so pull one at a +time. + +## Steps (network-volume mode) Only you can do the steps marked **(you)**. I do not enter credentials or launch pods. diff --git a/scripts/runpod/v41_reap/pod.sh b/scripts/runpod/v41_reap/pod.sh index 5ec4d09..f98ec30 100755 --- a/scripts/runpod/v41_reap/pod.sh +++ b/scripts/runpod/v41_reap/pod.sh @@ -6,9 +6,10 @@ # pod.sh pack CPU pod: build the calibration + eval pack from public datasets # pod.sh gpu 8xH200 pod: convert, teacher pass, candidate scoring; stops itself # pod.sh prune KEEP CPU pod: write the pruned HF checkpoint for the Mac to pull +# pod.sh all one 8xH200 pod does everything: fetch || pack, gpu, prune $PRUNE # -# Everything lives on one network volume mounted at /workspace, so stages can -# run on different (cheap CPU / expensive GPU) pods. +# Everything lives under /workspace (a network volume, or the pod's own volume +# disk in single-pod mode). set -euo pipefail MODEL_ID=deepseek-ai/DeepSeek-V4.1-Flash-1017 @@ -18,10 +19,18 @@ KIT=$W/keep/scripts/runpod/v41_reap HF=$W/v41-hf TP8=$W/v41-tp8 OUT=$W/out -CANDIDATES=${CANDIDATES:-"256 224 192 160"} +CANDIDATES=${CANDIDATES:-"256 192 128 96"} +PRUNE=${PRUNE:-"192 96"} # REAP 50% and REAP 75% of 384 experts MAX_GPU_HOURS=${MAX_GPU_HOURS:-6} # hard stop for the GPU pod (8xH200 secure ~ $42/h) export PYTHONPATH=$W/keep/src HF_HUB_ENABLE_HF_TRANSFER=1 +watchdog() { # whatever happens next, the pod stops at MAX_GPU_HOURS; a failed stage stops it at once + [[ -n "${WATCHDOG_STARTED:-}" ]] && return + export WATCHDOG_STARTED=1 + ( sleep $((MAX_GPU_HOURS * 3600)); stop_pod "watchdog: ${MAX_GPU_HOURS}h limit" ) & + trap 'status=$?; [[ $status -ne 0 ]] && stop_pod "stage failed (status $status)"' EXIT +} + log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$W/pod.log"; } stop_pod() { @@ -33,8 +42,13 @@ stop_pod() { fi } +fetch_small() { # tokenizer, configs, encoding/ and inference/: all pack and convert need besides weights + pip install -q "huggingface_hub[hf_transfer]>=0.34" "datasets>=3" tokenizers + hf download "$MODEL_ID" --revision "$REVISION" --local-dir "$HF" --include "*.json" "encoding/*" "inference/*" +} + fetch() { - pip install -q "huggingface_hub[hf_transfer]>=0.34" + fetch_small hf download "$MODEL_ID" --revision "$REVISION" --local-dir "$HF" python3 - "$HF" "$MODEL_ID" "$REVISION" <<'EOF' import hashlib, sys @@ -61,7 +75,6 @@ EOF } pack() { - pip install -q "datasets>=3" tokenizers huggingface_hub python3 -I "$KIT/build_pack.py" --model-dir "$HF" --out "$W/pack" log "pack built: $(python3 -c "import json;m=json.load(open('$W/pack/manifest.json'));print(m['records'],'records',m['sessions'],'sessions')")" } @@ -69,9 +82,7 @@ pack() { gpu() { local deadline deadline=$(date -u -d "+$((MAX_GPU_HOURS * 60 - 20)) minutes" +%FT%TZ) - # Watchdog: whatever happens below, the pod stops at MAX_GPU_HOURS. - ( sleep $((MAX_GPU_HOURS * 3600)); stop_pod "watchdog: ${MAX_GPU_HOURS}h limit" ) & - trap 'stop_pod "gpu stage exited (status $?)"' EXIT + watchdog cd "$HF/inference" pip install -q -r requirements.txt @@ -114,6 +125,20 @@ prune() { (cd "$W/v41-reap$keep" && sha256sum -c SHA256SUMS --quiet) && log "pruned checkpoint $W/v41-reap$keep verified" } -stage=${1:?usage: pod.sh fetch|pack|gpu|prune KEEP} +all() { + watchdog + fetch_small + log "fetch (weights) and pack in parallel" + fetch > "$W/fetch.log" 2>&1 & + local fetch_pid=$! + pack > "$W/pack.log" 2>&1 + wait "$fetch_pid" || { log "fetch failed: see $W/fetch.log"; return 1; } + gpu + rm -rf "$TP8" # 510 GB; the pruned checkpoints come from the HF source + for keep in $PRUNE; do prune "$keep"; done + log "all done: $(echo "$PRUNE" | sed "s|[0-9]*|$W/v41-reap&|g"). Pod left running for the Mac to pull; watchdog still armed." +} + +stage=${1:?usage: pod.sh fetch|pack|gpu|prune KEEP|all} shift "$stage" "$@" From 94b8cf58ad927f114f27aa5e523ae117099bb14c Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:30:09 -0800 Subject: [PATCH 19/74] runpod: document 8xH200 provisioning lessons in the V4.1 REAP runbook --- .../2026-11-05-v41-runpod-reap-runbook.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md index 19e3af2..21ed3a1 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md @@ -127,6 +127,66 @@ The pod stays up so the Mac can pull the results. The watchdog stops it at The Mac cannot hold both checkpoints at once (423 GB free), so pull one at a time. +## Getting an 8×H200 pod (lessons from 2026-11-06) + +No 8×H200 pod could be created on 2026-11-06, even though the catalog listed +stock. These are the lessons from that day. + +**Tools:** + +- **Use the official Runpod plugin.** Install it with + `claude plugin marketplace add runpod/runpod-plugins-official` and + `claude plugin install runpod@runpod`, then sign in through `/mcp` → + runpod. It signs in with OAuth, so no API key ends up on disk. Its MCP tools + are REST v2 and read live state. +- **Check stock for the GPU count you need:** + `get-gpu-type id="NVIDIA H200" include=AVAILABILITY product=POD count=8`. + This returns stock per data center, the host CUDA versions that have + capacity, the minimum RAM and vCPU, and `maxCount`. `runpodctl gpu list` is + not enough: its `stockStatus` is the best status across all data centers and + is counted for a single GPU. +- **Get the price and CUDA matrix for many types at once:** + `get-capacity gpuCount=8`. +- **Create the pod with REST v2 `create-pod`.** The body must include `disk`, + `mounts` or `templateId`. Without one of them you get *"You must either + provide a template id or pod configuration parameters"*. + +**What the catalog showed, and what happened:** + +- **Catalog:** 8×H200 secure was "Low", only in EUR-IS-5 and only on CUDA 13.2 + hosts, at $5.29/GPU/h ($42.32/h). Those hosts had 1510 GB RAM and 160 vCPU. + B200, H200 NVL and every H100 variant showed no 8-GPU stock. +- **Every H200 create failed with the same 400** (*"There are no longer any + instances available"*): + - with the host specs pinned (`allowedCudaVersions`, `minRamPerGpu`, + `minVcpuCountPerGpu`); + - with 1400 GB and 1100 GB volume disks; + - with community cloud; + - with no data-center filter; + - with a bare single H200. +- **The control worked.** A 1×RTX 4090 with the same image and ports was + created at once. So the account, the image (`runpod/pytorch:1.5.0-cu1281-torch291-ubuntu2404`) + and the request shape were all fine; H200 itself was unplaceable. + +**What to take from it:** + +- **Catalog stock is not a reservation.** A "Low" reading can be a host that is + already held. Runpod's create-pod docs say a create can fail for capacity on + a GPU the catalog just reported as available. +- **The 400 means one of two things.** It covers both a broken request and no + capacity, so separate them with a cheap control pod. If the control succeeds, + the request is fine and the problem is capacity. Disk size, ports and CUDA + pinning were not the cause. +- **Do not run tight retry loops against the API.** A loop of creates every + 90 s was stopped as spam. Check by hand from time to time, then create once. +- **The image does not limit placement.** A CUDA 12.8 image runs on CUDA 13.2 + hosts. + +**Fix the local Claude CLI first if plugin install fails.** Old npm installs of +`@anthropic-ai/claude-code` crash on Node 26. Remove them with +`npm uninstall -g --prefix @anthropic-ai/claude-code`, so that +`claude` resolves to a working install. + ## Steps (network-volume mode) Only you can do the steps marked **(you)**. I do not enter credentials or From 1f771cc281b517e04f9fe8e6662653231cf0e91b Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:38:53 -0800 Subject: [PATCH 20/74] docs: single-GPU layer-streamed design for the V4.1-Flash REAP run --- .../2026-11-06-v41-reap-single-gpu-design.md | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md diff --git a/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md b/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md new file mode 100644 index 0000000..429cfac --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md @@ -0,0 +1,153 @@ +# V4.1-Flash REAP on one GPU: design + +Date: 2026-11-06. This replaces the TP8 plan in +`2026-11-05-v41-runpod-reap-runbook.md`, which needed an 8×H200 pod that +Runpod could not place. Every figure below comes from the reference config and +code at revision `1bae0e10`. I fetched `inference/{config,model,kernel,convert,engram}.py`; +`model.py` has sha256 `e5decd76…8b8d2d`, which matches the pin in `collect.py`. + +## Why 8 GPUs was never about compute + +| Quantity | Value | +|---|---| +| Active parameters per token | about 15.6B: 374M per layer × 40 layers, plus the head | +| Forward cost | about 31 GFLOP per token | +| Total work | 3.2M calibration tokens, plus 0.45M eval tokens × (1 teacher + 4 students) ≈ 5.5M token passes ≈ 1.7e17 FLOP | +| Work on one H100 | 15 minutes at 200 TFLOPS effective; 1 hour at 50 TFLOPS | +| Weights | 510 GB on disk. Routed experts are 6.8 GB per layer in FP4, Engram tables are about 196 GB. | + +The 8×H200 pod existed only to hold all 510 GB in GPU memory at once. The work +itself fits on one GPU in under an hour. So the fix is to **stream weights +through one GPU, one layer at a time, over large token batches**. Holding every +weight resident is what forced the 8-GPU pod. + +## Design + +1. **Load layer weights straight from the HF shards.** There is no `convert.py` + step and no TP8 copy. With one GPU, `convert.py` only renames tensors, + dequantizes `wo_a` to bf16 and views the experts as `float4_e2m1fn_x2`. That + is about 20 lines, applied per tensor as it loads. This saves the hour of + conversion, the 510 GB TP8 copy, and the need for more than 550 GB of RAM. +2. **Run the reference code unchanged, one layer at a time.** Build the + reference `Block` modules with `world_size=1`. For each chunk of records + (whole records, so attention sees full context): + - keep the chunk's hyper-connection residual in GPU memory, as + `[tokens, 4, 5120]` bf16 (40 KB per token); + - for each layer, load its weights, prefetching the next layer's; + - run attention one micro-batch of records at a time; + - run the MoE over the whole chunk at once, so each expert sees about 15K + tokens instead of about 100, which makes the expert GEMMs efficient. + + The REAP, imatrix and route hooks in `make_moe_forward` stay as they are. +3. **Save and restore the cross-layer attention state.** Layers pass + compressed KV, indexer keys, top-k indices and candidate blocks to later + layers through the module-level `shared_attn` object. KV comes from source + layers 2, 8, 14 and 20; the indexer from 2, 8, 14, 20, 24, 28, 32 and 36; + candidates from layer 20 into every layer after it. Done one layer at a + time, that state has to be saved per micro-batch and restored before the + next layer. The candidate mask is the large part, up to about 16 KB per + token at 16K context. It gets bit-packed, or kept in host RAM. +4. **Precompute the Engram lookups during the pack stage, on the CPU.** The + rows Engram fetches depend only on the token ids + (`NgramHashState` → `embed(hash_ids)`); the gate needs the hidden state, but + the lookup does not. Gathering the rows once per token (24 × 256 values, + about 6.5 KB per token per Engram layer, about 48 GB in all) means the GPU + never touches the 196 GB tables, and the GPU pass makes no random IO. +5. **Do the whole job in two sweeps over the weights.** + - **Sweep A, calibration** (4 chunks on an 80 GB GPU, 2 on an H200): writes + the REAP stats, imatrix and routes, then `finalize-reap` and + `reap-select` produce the survivors for 256/192/128/96. + - **Sweep B, eval:** the teacher and the 4 masked-router students run side + by side through each layer, so each layer is read once for all 5 streams. + This writes the teacher's top-2048 logits and the student + KL / NLL change / top-1 agreement, in the same npz schema as now. That + replaces the separate student `torchrun` pass. + + Weight IO is about 320 GB per sweep once Engram is precomputed: 2-3 minutes + from NVMe, overlapped with compute. +6. **Use as many GPUs as are free, data-parallel by chunk.** Chunks are + independent, so any pod with 1 to 8 GPUs of the Hopper or Blackwell + class (80 GB or more) works. The stats are sums, so the shards add together + exactly as the per-session npz files do today. +7. **Skip the MTP drafter.** It isn't needed for backbone REAP, and the pruner + already leaves `mtp.*` alone. + +## REAP-specific streamlining + +- **Calibration budget:** the per-session npz files already let me compute a + split-half survivor Jaccard offline, at no extra cost. If two halves of the + calibration set pick nearly the same survivors (Jaccard ≥ 0.98 at N=96), then + 3.2M tokens is more than needed and the next run can use fewer. +- **Candidates:** all four are cheap now. Each extra student costs memory + during Sweep B, not another pass over the weights. +- **E8P imatrix from pruned routing:** the teacher imatrix counts traffic + under the full 384-expert routing. After pruning, the surviving experts take + over the pruned experts' tokens. Sweep B can also write a pruned-routing + imatrix for 192 and 96 at no extra cost. The catch is that it comes from the + eval split, so it would leak a little into the KL measurement if it were + used to fit E8P. I'll write it, but not use it by default. + +## What changes in the kit + +| File | Change | +|---|---| +| `collect.py` | Replace the `torchrun` driver with the one-layer-at-a-time driver. Keep `SessionStats`, the MoE hook, the head capture and the npz writers. | +| `build_pack.py` | Add the Engram row precompute. | +| `pod.sh` | Drop `convert` and the TP8 directory. `gpu` becomes sweep A, select, then sweep B. | +| `prune.py`, `score.py`, `finalize-reap`, `reap-select` | No change. | + +## Validation before any spend + +1. **Run the real `model.py` on the CPU, at tiny size.** Replace + `kernel.py`'s tilelang functions with plain-torch versions (`act_quant`, + `fp4_act_quant`, `fp8_gemm`, `fp4_gemm`, `sparse_attn`, + `hc_split_sinkhorn`; the repo already has a torch Sinkhorn). Use a tiny + random `ModelArgs` that still has hyper-connections, KV and indexer sources, + a candidate source and Engram. The test then checks that the + one-layer-at-a-time driver matches `Transformer.forward` logits and REAP + stats with: + - one record per micro-batch; + - several records per micro-batch; + - records split across chunks. + + This is the test that catches mistakes in saving and restoring + `shared_attn`. +2. **First 10 minutes on the pod:** run the same equivalence test with the real + tilelang kernels at tiny size, then one real 16K record through all 40 + layers. Check the teacher NLL is sensible, and measure GPU memory and tok/s. + Stop if anything is off. + +## Cost + +These prices are from the Runpod catalog read of 2026-11-06. 1×H100 SXM costs +$2.69/h, and 1×H200 costs $3.59/h on community cloud or $5.29/h on secure. + +| Stage | Time on 1×H100 | +|---|---| +| fetch and pack, in parallel | 1-3 h | +| Engram precompute | under 1 h | +| sweeps A and B | about 1-2 h | +| prune 192 and 96 | about 1 h | + +The total is about 4-7 h, roughly **$15-25**, against $125-340 for the TP8 +plan. The disk needed is about 1.3 TB: + +| Item | Size | +|---|---| +| HF source | 510 GB | +| Engram rows | 48 GB | +| out | about 30 GB | +| REAP-192 | 366 GB | +| REAP-96 | 293 GB | + +## Risks + +- **GPU kernels:** the tilelang kernels are written for Hopper and Blackwell. + H100 and H200 are the safe target. The RTX PRO 6000 (SM120) is not verified. +- **`shared_attn` state:** saving and restoring it is the subtle part, and the + CPU equivalence test covers it. +- **Engram precompute:** this relies on `embed(hash_ids)` being a pure row + gather. I checked the code: `ParallelEngramEmbedding.forward` gathers FP8 + rows and their scales, then dequantizes. The pack can therefore store the + raw FP8 rows and scales (about 6.3 KB per token per layer), and the driver + dequantizes them exactly as the module does. From 0622bc25dcd064398e086750fdc6d0fc9b296c93 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:59:31 -0800 Subject: [PATCH 21/74] runpod: layer-streamed single-GPU driver for the V4.1-Flash REAP run The reference Block modules now run one layer at a time over chunks of whole records, loading each layer straight from the HF shards. No convert.py, no TP8 copy, and any 80 GB+ GPU works. - stream.py: HFCheckpoint (per-layer loads with convert.py's fixups), EngramRows (host memmap gather of the FP8 tables), LayerStreamer (per-record save and restore of shared_attn, candidates bit-packed). - collect.py: --sweep calibration|eval, --shard I/N, --chunk-tokens. The eval sweep runs the teacher and every masked-router student side by side. The npz schemas are unchanged. - pod.sh: one collect.py per GPU per sweep; the convert step and the TP8 directory are gone. - tests: the pinned model.py and engram.py run on CPU at toy size with plain-torch kernels. The streamed forward matches Transformer.forward bit for bit. Calibration stats, teacher logits and student scores match the reference. --- .../2026-11-05-v41-runpod-reap-runbook.md | 4 + .../2026-11-06-v41-reap-single-gpu-design.md | 60 +- scripts/runpod/v41_reap/collect.py | 449 +++--- scripts/runpod/v41_reap/pod.sh | 43 +- scripts/runpod/v41_reap/stream.py | 333 +++++ scripts/runpod/v41_reap/torch_kernel.py | 62 + tests/fixtures/v41_reference/LICENSE | 21 + tests/fixtures/v41_reference/README.md | 3 + tests/fixtures/v41_reference/engram.py | 184 +++ tests/fixtures/v41_reference/model.py | 1312 +++++++++++++++++ tests/test_runpod_v41_stream.py | 266 ++++ 11 files changed, 2508 insertions(+), 229 deletions(-) create mode 100644 scripts/runpod/v41_reap/stream.py create mode 100644 scripts/runpod/v41_reap/torch_kernel.py create mode 100644 tests/fixtures/v41_reference/LICENSE create mode 100644 tests/fixtures/v41_reference/README.md create mode 100644 tests/fixtures/v41_reference/engram.py create mode 100644 tests/fixtures/v41_reference/model.py create mode 100644 tests/test_runpod_v41_stream.py diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md index 21ed3a1..82d3c27 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md @@ -2,6 +2,10 @@ Date: 2026-11-05. Budget: $497. Kit: `scripts/runpod/v41_reap/`. +> **2026-11-06:** the GPU stage now runs on one GPU (or several, data-parallel), +> streaming the weights one layer at a time with no `convert.py` step. See +> `2026-11-06-v41-reap-single-gpu-design.md`. The 8×H200 / TP8 steps below are kept for the record. + ## Goal The V4-Flash artifacts are gone. This run makes new V4.1 artifacts on rented diff --git a/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md b/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md index 429cfac..963c3cc 100644 --- a/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md +++ b/docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md @@ -47,12 +47,15 @@ weight resident is what forced the 8-GPU pod. time, that state has to be saved per micro-batch and restored before the next layer. The candidate mask is the large part, up to about 16 KB per token at 16K context. It gets bit-packed, or kept in host RAM. -4. **Precompute the Engram lookups during the pack stage, on the CPU.** The - rows Engram fetches depend only on the token ids - (`NgramHashState` → `embed(hash_ids)`); the gate needs the hidden state, but - the lookup does not. Gathering the rows once per token (24 × 256 values, - about 6.5 KB per token per Engram layer, about 48 GB in all) means the GPU - never touches the 196 GB tables, and the GPU pass makes no random IO. +4. **Gather the Engram rows on the host, at run time.** The rows Engram + fetches depend only on the token ids (`NgramHashState` → `embed(hash_ids)`); + the gate needs the hidden state, but the lookup does not. A stand-in for + `ParallelEngramEmbedding` memory-maps the two 196 GB tables straight from the + shard files, gathers the FP8 rows and scales a chunk needs (24 × 256 values + per token per Engram layer) and dequantizes them exactly as the module does. + The tables never reach the GPU. I first planned to precompute the rows in + the pack stage (about 48 GB on disk); the run-time gather does the same + thing with no extra stage and no extra disk. 5. **Do the whole job in two sweeps over the weights.** - **Sweep A, calibration** (4 chunks on an 80 GB GPU, 2 on an H200): writes the REAP stats, imatrix and routes, then `finalize-reap` and @@ -63,8 +66,12 @@ weight resident is what forced the 8-GPU pod. KL / NLL change / top-1 agreement, in the same npz schema as now. That replaces the separate student `torchrun` pass. - Weight IO is about 320 GB per sweep once Engram is precomputed: 2-3 minutes - from NVMe, overlapped with compute. + Every chunk is one pass over the backbone weights, about 320 GB without the + Engram tables. A chunk holds `--chunk-tokens` resident tokens (default 512K, + counting each stream), so one 80 GB GPU makes about 7 calibration passes and + 5 eval passes. That is about 4 TB of reads. From NVMe this is 20-30 minutes, + overlapped with compute by a one-layer prefetch, and less when the pod's + page cache holds the shards. 6. **Use as many GPUs as are free, data-parallel by chunk.** Chunks are independent, so any pod with 1 to 8 GPUs of the Hopper or Blackwell class (80 GB or more) works. The stats are sums, so the shards add together @@ -91,9 +98,11 @@ weight resident is what forced the 8-GPU pod. | File | Change | |---|---| -| `collect.py` | Replace the `torchrun` driver with the one-layer-at-a-time driver. Keep `SessionStats`, the MoE hook, the head capture and the npz writers. | -| `build_pack.py` | Add the Engram row precompute. | -| `pod.sh` | Drop `convert` and the TP8 directory. `gpu` becomes sweep A, select, then sweep B. | +| `stream.py` (new) | `HFCheckpoint` (per-layer loads with `convert.py`'s fixups), `EngramRows` (host gather), `LayerStreamer` (the reference `Block`s one layer at a time, with `shared_attn` saved and restored per record). | +| `torch_kernel.py` (new) | Plain-torch stand-ins for `kernel.py`, for the CPU tests. | +| `collect.py` | The `torchrun` driver is replaced by `--sweep calibration|eval`, `--shard I/N` and `--chunk-tokens`. `SessionStats`, the MoE hook, the head capture and the npz schemas are unchanged. | +| `build_pack.py` | No change. | +| `pod.sh` | `convert` and the TP8 directory are gone. `gpu` runs one `collect.py` per GPU for sweep A, then select, then sweep B. | | `prune.py`, `score.py`, `finalize-reap`, `reap-select` | No change. | ## Validation before any spend @@ -112,6 +121,19 @@ weight resident is what forced the 8-GPU pod. This is the test that catches mistakes in saving and restoring `shared_attn`. + + **Status (2026-11-06): done.** `tests/test_runpod_v41_stream.py` runs the + pinned `model.py` and `engram.py` (copied to `tests/fixtures/v41_reference/`) + on a 6-layer toy with every feature above. The streamed hidden state equals + `Transformer.forward` bit for bit, with one MoE batch per chunk and with one + per record, and a record's output does not depend on its chunk-mates. Breaking + any one `shared_attn` field, the clone of block buffers, or the Engram lookup + fails the test. Both sweeps are checked too: + - calibration REAP, imatrix and route stats match the reference per session + (route counts exactly, sums to 1e-6); + - eval teacher logits match; + - a keep-all student scores KL 0; + - a keep-4-of-8 student's NLL matches the reference with that router mask. 2. **First 10 minutes on the pod:** run the same equivalence test with the real tilelang kernels at tiny size, then one real 16K record through all 40 layers. Check the teacher NLL is sensible, and measure GPU memory and tok/s. @@ -125,17 +147,15 @@ $2.69/h, and 1×H200 costs $3.59/h on community cloud or $5.29/h on secure. | Stage | Time on 1×H100 | |---|---| | fetch and pack, in parallel | 1-3 h | -| Engram precompute | under 1 h | | sweeps A and B | about 1-2 h | | prune 192 and 96 | about 1 h | -The total is about 4-7 h, roughly **$15-25**, against $125-340 for the TP8 -plan. The disk needed is about 1.3 TB: +The total is about 3-6 h, roughly **$10-25**, against $125-340 for the TP8 +plan. The disk needed is about 1.2 TB: | Item | Size | |---|---| | HF source | 510 GB | -| Engram rows | 48 GB | | out | about 30 GB | | REAP-192 | 366 GB | | REAP-96 | 293 GB | @@ -146,8 +166,10 @@ plan. The disk needed is about 1.3 TB: H100 and H200 are the safe target. The RTX PRO 6000 (SM120) is not verified. - **`shared_attn` state:** saving and restoring it is the subtle part, and the CPU equivalence test covers it. -- **Engram precompute:** this relies on `embed(hash_ids)` being a pure row +- **Engram host gather:** this relies on `embed(hash_ids)` being a pure row gather. I checked the code: `ParallelEngramEmbedding.forward` gathers FP8 - rows and their scales, then dequantizes. The pack can therefore store the - raw FP8 rows and scales (about 6.3 KB per token per layer), and the driver - dequantizes them exactly as the module does. + rows and their scales, then dequantizes, and the CPU test runs the real + module's math through the stand-in. +- **Indexer memory at 16K context:** the indexer's `bshd,btd->bsht` einsum is + about 16 GB for one 16K record. It should fit on 80 GB, but the first pod + minutes must check `max_gpu_gb` in the progress log before the full run. diff --git a/scripts/runpod/v41_reap/collect.py b/scripts/runpod/v41_reap/collect.py index 41e1e00..052dccd 100644 --- a/scripts/runpod/v41_reap/collect.py +++ b/scripts/runpod/v41_reap/collect.py @@ -1,8 +1,11 @@ -"""Teacher pass for DeepSeek-V4.1-Flash on a GPU node: REAP, imatrix, routes, logits. +"""REAP, imatrix, routes and teacher logits for DeepSeek-V4.1-Flash, on one GPU. -Runs DeepSeek's own reference inference code (``inference/model.py`` at the -pinned revision) under ``torchrun``, with one change: ``MoE.forward`` is -replaced by a copy that also records, for every routed slot, +DeepSeek's own reference code (``inference/model.py`` at the pinned revision) +runs one layer at a time through ``stream.LayerStreamer``. Weights come straight +from the HF shards (no ``convert.py``), and any GPU of 80 GB or more works. +Design: ``docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md``. + +``MoE.forward`` is replaced by a copy that also records, for every routed slot, - REAP statistics (``keep.quality.reap.ReapStats`` arrays, ``reap__*``); - imatrix sums for the two expert input spaces (``hidden`` = the post @@ -11,28 +14,32 @@ - the first ``ROUTE_RECORD_TOKENS`` routes of each session (``routes__*``), for ``benchmarks/replay_dsv4_expert_residency.py``. -Eval sessions also capture top-K teacher logits at their supervised positions, -in the teacher runner's ``logits`` schema. Output, per session, is one npz under -``//sessions/``, so KEEP's existing ``finalize-reap`` and imatrix -finalizers read it unchanged. +Two sweeps over the weights: + +- ``--sweep calibration`` writes REAP + imatrix + routes per calibration session. +- ``--sweep eval`` runs the teacher and every ``--survivors`` student side by + side. The teacher writes REAP + routes and top-K logits per eval session. Each + student is scored against the teacher's top-K (KL, NLL, top-1 agreement) and + written to ``/student-KEEP/``. + +Output is one npz per session under ``//sessions/``, so KEEP's +``finalize-reap``, the imatrix finalizers and ``score.py`` read it unchanged. The replacement forward computes exactly what the reference does (same ops, -same order); ``tests/test_runpod_v41_reap.py`` checks it, bit for bit, -against a CPU copy of the reference Gate / Expert / MoE. One approximation: the reference folds the gate weight -in *before* w2 (``w2(g * h)``), so the expert output norm ``||f||`` is taken as -``||w2(g * h)|| / g``. With FP8 activation quantization in w2 that differs -from ``||w2(h)||`` by rounding only. - -Usage (on the pod, from the reference ``inference`` directory):: - - torchrun --nproc-per-node 8 /workspace/keep/scripts/runpod/v41_reap/collect.py \\ - --ckpt-path /workspace/v41-tp8 --config config.json \\ - --pack /workspace/pack/v41-pack.jsonl --out /workspace/out \\ - --deadline-utc 2026-11-07T06:00:00Z - -Then, after finalize-reap and reap-select on the teacher pass, the same command -with ``--survivors 192=/workspace/out/survivors-192.json`` (repeatable) scores -each candidate expert count against the teacher in ``/student-KEEP/``. +same order). ``tests/test_runpod_v41_reap.py`` checks it bit for bit against a +CPU copy of the reference Gate / Expert / MoE, and ``tests/test_runpod_v41_stream.py`` +checks both sweeps against ``Transformer.forward``. One approximation: the +reference folds the gate weight in *before* w2 (``w2(g * h)``), so the expert +output norm ``||f||`` is taken as ``||w2(g * h)|| / g``. With FP8 activation +quantization in w2 that differs from ``||w2(h)||`` by rounding only. + +Usage, on the pod, one process per GPU (``--shard I/N`` splits the sessions):: + + CUDA_VISIBLE_DEVICES=0 python3 collect.py --hf-path /workspace/v41-hf \\ + --pack /workspace/pack/v41-pack.jsonl --out /workspace/out --sweep calibration --shard 0/1 + +then finalize-reap and reap-select, then the same command with ``--sweep eval`` +and ``--survivors 192=/workspace/out/survivors-192.json`` (repeatable). """ from __future__ import annotations @@ -41,7 +48,6 @@ import datetime as dt import hashlib import json -import os import sys import time from collections import OrderedDict @@ -107,8 +113,8 @@ def add_routes(self, layer: int, indices: Any, weights: Any, *, record: bool) -> rows, top_k = indices.shape if record: # rank 0 only: identical on every rank self.routed_slots[layer] += rows * top_k - for space in SPACES: - self.imatrix[space]["total_route_count"][layer] += rows * top_k + for m in self.imatrix.values(): + m["total_route_count"][layer] += rows * top_k room = ROUTE_RECORD_TOKENS - self.route_tokens[layer] if room > 0: self.route_ids[layer].append(indices[:room].to("cpu").numpy().astype(np.int16)) @@ -132,6 +138,8 @@ def add_expert(self, layer: int, expert: int, x: Any, gate: Any, out: Any, h: An abs_max = (out64.abs().amax(dim=-1) / g).max() r["abs_max"][layer, expert] = torch.maximum(r["abs_max"][layer, expert], abs_max) for space, rows in (("hidden", x), ("down", h)): + if space not in self.imatrix: + continue sq = rows.double().square() m = self.imatrix[space] m["importance_sum"][layer, expert] += sq.sum(dim=0) @@ -146,8 +154,8 @@ def reduce(self, dist: Any) -> None: op = dist.ReduceOp.MAX if name in ("norm_max", "abs_max") else dist.ReduceOp.SUM dist.all_reduce(value, op=op) dist.all_reduce(self.routed_slots) - for space in SPACES: - for value in self.imatrix[space].values(): + for m in self.imatrix.values(): + for value in m.values(): dist.all_reduce(value) def arrays(self, layers: list[int]) -> dict[str, np.ndarray]: @@ -162,8 +170,7 @@ def arrays(self, layers: list[int]) -> dict[str, np.ndarray]: for name, value in self.reap.items(): arr = np_(value) out[f"{REAP_PREFIX}{name}"] = np.rint(arr).astype(np.int64) if name == "route_count" else arr - for space in SPACES: - m = self.imatrix[space] + for space, m in self.imatrix.items(): # float32 on disk, as KEEP's validator requires (sums stay float64 until here) out[f"importance_sum__{space}"] = np_(m["importance_sum"]).astype(np.float32) out[f"affinity_weighted_importance__{space}"] = np_(m["affinity_weighted_importance"]).astype(np.float32) @@ -343,6 +350,135 @@ def save_npz(path: Path, arrays: dict[str, np.ndarray]) -> None: tmp.replace(path) +# --------------------------------------------------------------------------- +# Sweeps (stream.LayerStreamer: one GPU, one layer at a time) +# --------------------------------------------------------------------------- + +TEACHER = "teacher" + + +def chunk_sessions(sessions: list[tuple[str, list[dict[str, Any]]]], budget: int, streams: int) -> list[list[tuple[str, list[dict[str, Any]]]]]: + """Whole sessions, in order, up to ``budget`` resident tokens (tokens x streams) per chunk.""" + + chunks: list[list[tuple[str, list[dict[str, Any]]]]] = [] + size = 0 + for name, records in sessions: + n = sum(len(r["token_ids"]) for r in records) * streams + if not chunks or (chunks[-1] and size + n > budget): + chunks.append([]) + size = 0 + chunks[-1].append((name, records)) + size += n + return chunks + + +def survivor_masks(table: dict[str, Any], n_experts: int) -> dict[int, Any]: + """reap-select table -> {layer: bool tensor, True = pruned}. Masking the router to -inf + equals slicing it (tests/test_reap.py), so one set of weights scores every candidate.""" + + import torch + + masks = {} + for layer, kept in table["survivors"].items(): + mask = torch.ones(n_experts, dtype=torch.bool) + mask[torch.as_tensor(kept)] = False + masks[int(layer)] = mask + return masks + + +def install_hooks(streamer: Any, *, top_k: int, rows_per_slice: int) -> dict[str, Any]: + """Patch the reference MoE and the streamer's head once; returns the shared ``state``.""" + + import torch + import torch.nn.functional as F + + state: dict[str, Any] = {"rank": 0, "stats": None, "positions": None, "teacher": None} + streamer.ref.MoE.forward = make_moe_forward(streamer.ref, torch, F, None, state) + streamer.head.forward = make_head_forward(streamer.head, torch, F, None, state, top_k=top_k, rows_per_slice=rows_per_slice) + return state + + +def run_chunk( + streamer: Any, + state: dict[str, Any], + chunk: list[tuple[str, list[dict[str, Any]]]], + *, + split: str, + students: dict[str, dict[int, Any]], + dims: dict[str, int], + device: Any, +) -> dict[tuple[str, str], dict[str, np.ndarray]]: + """One pass over the weights for ``chunk``; returns arrays per (stream, session). + + Calibration: the teacher only, REAP + imatrix + routes per session. + Eval: the teacher (REAP + routes + top-K logits) and every student (scored against + the teacher) side by side. Teacher records go through the MoE per session, so the + stats stay per session; each student's records go through it together. + """ + + import torch + from stream import Item + + args = streamer.args + n_layers, n_experts = int(args.n_layers), int(args.n_routed_experts) + stats = { + name: SessionStats(torch=torch, n_layers=n_layers, n_experts=n_experts, dims=dims if split == "calibration" else {}, device=device) + for name, _ in chunk + } + items: list[Item] = [] + streams_of: dict[str, list[tuple[str, Item]]] = {} + for name, records in chunk: + for record in records: + tokens = torch.as_tensor([record["token_ids"]], dtype=torch.long, device=device) + for stream in (TEACHER, *students) if split == "eval" else (TEACHER,): + item = Item(tokens=tokens, group=(TEACHER, name) if stream == TEACHER else (stream, None)) + items.append(item) + streams_of.setdefault(name, []).append((stream, item)) + + def before_moe(layer: int, block: Any, key: tuple[str, str | None]) -> None: + stream, name = key + bias = block.ffn.gate.bias + if not hasattr(block, "full_bias"): + block.full_bias = bias.data.clone() + bias.data.copy_(block.full_bias) + mask = students[stream].get(layer) if stream != TEACHER else None + if mask is not None: + bias.data[mask.to(bias.device)] = float("-inf") + state["stats"] = stats[name] if stream == TEACHER and layer < n_layers else None + + with torch.inference_mode(): + streamer.forward(items, before_moe) + state["stats"] = None + + layers = list(range(n_layers)) + results: dict[tuple[str, str], dict[str, np.ndarray]] = {} + for name, records in chunk: + results[(TEACHER, name)] = stats[name].arrays(layers) + if split != "eval": + continue + (record,) = records + positions = target_positions(record) + targets = np.asarray(record["token_ids"], dtype=np.int64)[positions + 1] + state["positions"] = torch.as_tensor(positions, device=device) + state["targets"] = torch.as_tensor(targets, device=device) + base = {"positions": positions.astype(np.int32), "target_token_ids": targets.astype(np.int32)} + state["teacher"] = None + for stream, item in streams_of[name]: # the teacher first, then the students it scores + state["captured"] = {k: [] for k in CAPTURE_KEYS} + with torch.inference_mode(): + streamer.head(streamer.final_hidden(item)) + cap = {k: np.concatenate(v) for k, v in state["captured"].items() if v} + results.setdefault((stream, name), {}).update(base | cap) + if stream == TEACHER: + state["teacher"] = { # the fp16 values the npz keeps, as the old two-pass flow read them + "ids": torch.as_tensor(cap["topk_logit_ids"].astype(np.int64), device=device), + "logprob": torch.as_tensor(cap["topk_logit_values"].astype(np.float32) - cap["logsumexp"][:, None], device=device), + "targets": state["targets"], + } + state["positions"] = state["targets"] = state["teacher"] = None + return results + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -350,84 +486,65 @@ def save_npz(path: Path, arrays: dict[str, np.ndarray]) -> None: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--ckpt-path", required=True, help="convert.py output: model{rank}-mp{N}.safetensors + tokenizer") - parser.add_argument("--config", default="config.json", help="reference inference/config.json") + parser.add_argument("--hf-path", required=True, type=Path, help="the pinned HF checkpoint (safetensors shards + tokenizer)") + parser.add_argument("--ref-dir", type=Path, help="reference inference/ directory (default: HF_PATH/inference)") parser.add_argument("--pack", required=True, type=Path) parser.add_argument("--out", required=True, type=Path) - parser.add_argument("--max-seq-len", type=int, default=16384) - parser.add_argument("--top-k", type=int, default=2048, help="teacher logits kept per eval position") - parser.add_argument("--rows-per-slice", type=int, default=256) - parser.add_argument("--deadline-utc", help="stop cleanly after the session running at this time") + parser.add_argument("--sweep", required=True, choices=("calibration", "eval")) parser.add_argument( "--survivors", action="append", default=[], metavar="KEEP=PATH", - help="student pass (repeatable): mask the router to a reap-select table, score eval sessions " - "against the teacher run in --out, write /student-KEEP/", + help="eval sweep (repeatable): also run a student with the router masked to this reap-select " + "table, scored against the teacher, into /student-KEEP/", ) + parser.add_argument("--chunk-tokens", type=int, default=524288, help="resident tokens per weight pass (x streams)") + parser.add_argument("--shard", default="0/1", help="I/N: this process takes every Nth session from I") + parser.add_argument("--max-seq-len", type=int, default=16384) + parser.add_argument("--top-k", type=int, default=2048, help="teacher logits kept per eval position") + parser.add_argument("--rows-per-slice", type=int, default=256) + parser.add_argument("--deadline-utc", help="stop cleanly before the first chunk that starts after this time") parser.add_argument("--skip-hash-check", action="store_true", help="tests only") args = parser.parse_args(argv) + if args.survivors and args.sweep != "eval": + parser.error("--survivors needs --sweep eval") + shard, n_shards = map(int, args.shard.split("/")) import torch - import torch.distributed as dist - import torch.nn.functional as F - from safetensors.torch import load_model from transformers import AutoTokenizer - here = Path.cwd() - model_py = here / "model.py" - digest = hashlib.sha256(model_py.read_bytes()).hexdigest() + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import stream + + ref_dir = args.ref_dir or args.hf_path / "inference" + digest = hashlib.sha256((ref_dir / "model.py").read_bytes()).hexdigest() if not args.skip_hash_check and digest != REFERENCE_MODEL_PY_SHA256: - raise SystemExit(f"{model_py} sha256 {digest} is not the pinned reference ({REFERENCE_MODEL_PY_SHA256})") - sys.path.insert(0, str(here)) - import model as ref - - world_size = int(os.getenv("WORLD_SIZE", "1")) - rank = int(os.getenv("RANK", "0")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - if world_size > 1: - dist.init_process_group("nccl") - torch.cuda.set_device(local_rank) + raise SystemExit(f"{ref_dir / 'model.py'} sha256 {digest} is not the pinned reference ({REFERENCE_MODEL_PY_SHA256})") + ref = stream.import_reference(ref_dir) + torch.cuda.memory._set_allocator_settings("expandable_segments:True") torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") torch.manual_seed(33377335) - - with open(args.config) as f: - config = json.load(f) - model_args = ref.ModelArgs(**config) + model_args = ref.ModelArgs(**json.loads((ref_dir / "config.json").read_text())) model_args.max_batch_size = 1 model_args.max_seq_len = args.max_seq_len - tokenizer = AutoTokenizer.from_pretrained(args.ckpt_path) - with torch.device("cuda"): - model = ref.Transformer(model_args, tokenizer) - load_model(model, os.path.join(args.ckpt_path, f"model{rank}-mp{world_size}.safetensors")) - torch.set_default_device("cuda") - model.eval() - original_bias = { - (layer, leaf): getattr(block.ffn.gate, leaf).data.clone() - for layer, block in enumerate(model.layers) - for leaf in ("bias", "bias_vl") - if getattr(block.ffn.gate, leaf) is not None - } - def mask_router(table_path: Path | None) -> str | None: - """Masking the router to -inf equals slicing it (tests/test_reap.py), so one - loaded checkpoint scores every candidate expert count.""" - - for (layer, leaf), value in original_bias.items(): - getattr(model.layers[layer].ffn.gate, leaf).data.copy_(value) - if table_path is None: - return None - for layer, kept in json.loads(table_path.read_text())["survivors"].items(): - gate = model.layers[int(layer)].ffn.gate - mask = torch.ones(gate.bias.shape[0], dtype=torch.bool, device=gate.bias.device) - mask[torch.as_tensor(kept, device=mask.device)] = False - for leaf in ("bias", "bias_vl"): - if (int(layer), leaf) in original_bias: - getattr(gate, leaf).data[mask] = float("-inf") - return hashlib.sha256(table_path.read_bytes()).hexdigest() - - state: dict[str, Any] = {"rank": rank, "stats": None, "positions": None} - ref.MoE.forward = make_moe_forward(ref, torch, F, dist, state) - model.head.forward = make_head_forward(model.head, torch, F, dist, state, top_k=args.top_k, rows_per_slice=args.rows_per_slice) + ckpt = stream.HFCheckpoint(args.hf_path) + layout = ref.EngramLayout.from_args(model_args) + hash_state = None + if layout is not None: + with torch.device("cuda"): + hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) + engram_rows = {layer: stream.EngramRows.from_checkpoint(ckpt, layer, ref.fp8_block_size) for layer in model_args.engram_layer_ids} + streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, "cuda") + state = install_hooks(streamer, top_k=args.top_k, rows_per_slice=args.rows_per_slice) + + n_experts = int(model_args.n_routed_experts) + students: dict[str, dict[int, Any]] = {} + survivors_sha: dict[str, str] = {} + for spec in args.survivors: + keep, _, table = spec.partition("=") + students[keep] = survivor_masks(json.loads(Path(table).read_text()), n_experts) + survivors_sha[keep] = hashlib.sha256(Path(table).read_bytes()).hexdigest() run_config = { "model_id": MODEL_ID, @@ -436,115 +553,65 @@ def mask_router(table_path: Path | None) -> str | None: "pack_sha256": hashlib.sha256(args.pack.read_bytes()).hexdigest(), "max_seq_len": args.max_seq_len, "top_k": args.top_k, - "world_size": world_size, + "driver": "layer-streamed, world_size 1", } + text = json.dumps(run_config, indent=2, sort_keys=True) + "\n" + config_path = args.out / "run-config.json" + args.out.mkdir(parents=True, exist_ok=True) + if not config_path.exists() or config_path.read_text() != text: + config_path.write_text(text) config_sha = hashlib.sha256(json.dumps(run_config, sort_keys=True).encode()).hexdigest() - if rank == 0: - args.out.mkdir(parents=True, exist_ok=True) - (args.out / "run-config.json").write_text(json.dumps(run_config, indent=2, sort_keys=True) + "\n") - + teacher_config_sha = hashlib.sha256(text.encode()).hexdigest() + + def target(stream_name: str, name: str) -> Path: + root = args.out if stream_name == TEACHER else args.out / f"student-{stream_name}" + return root / args.sweep / "sessions" / f"{name}.npz" + + streams = (TEACHER, *students) + todo = [ + (name, records) + for i, (name, records) in enumerate(item for item in load_sessions(args.pack).items() if item[1][0]["split"] == args.sweep) + if i % n_shards == shard and not all(target(s, name).exists() for s in streams) + ] + for name, records in todo: + if any(len(r["token_ids"]) > args.max_seq_len for r in records): + raise ValueError(f"session {name} has a record over --max-seq-len {args.max_seq_len}") deadline = None if args.deadline_utc: deadline = dt.datetime.fromisoformat(args.deadline_utc.replace("Z", "+00:00")).timestamp() - n_layers = int(model_args.n_layers) - n_experts = int(model_args.n_routed_experts) dims = {"hidden": int(model_args.dim), "down": int(model_args.moe_inter_dim)} - layers = list(range(n_layers)) - sessions = load_sessions(args.pack) started = time.time() - teacher_config_sha = hashlib.sha256((args.out / "run-config.json").read_bytes()).hexdigest() if rank == 0 else "" - - def run_pass(out_dir: Path, survivors_sha: str | None) -> bool: - """One pass over the pack. Teacher pass when ``survivors_sha`` is None, else a - masked-router student pass over the eval sessions. False once the deadline hits.""" - - student = survivors_sha is not None - progress = out_dir / "progress.jsonl" - for name, records in sessions.items(): - split = records[0]["split"] - target = out_dir / split / "sessions" / f"{name}.npz" - if target.exists() or (student and split != "eval"): - continue - if deadline is not None and time.time() > deadline: - if rank == 0: - print(f"deadline reached before session {name}; stopping cleanly", flush=True) - return False - t0 = time.time() - stats = None if student else SessionStats(torch=torch, n_layers=n_layers, n_experts=n_experts, dims=dims, device="cuda") - state["stats"] = stats - state["captured"] = {k: [] for k in CAPTURE_KEYS} - if student: - with np.load(args.out / "eval" / "sessions" / f"{name}.npz") as teacher: - state["teacher"] = { - "ids": torch.as_tensor(teacher["topk_logit_ids"].astype(np.int64), device="cuda"), - "logprob": torch.as_tensor( - teacher["topk_logit_values"].astype(np.float32) - teacher["logsumexp"][:, None], device="cuda" - ), - "targets": torch.as_tensor(teacher["target_token_ids"].astype(np.int64), device="cuda"), - } - tokens_seen = 0 - logits_arrays: dict[str, np.ndarray] = {} - for record in records: - ids = record["token_ids"] - if len(ids) > args.max_seq_len: - raise ValueError(f"{record['id']} has {len(ids)} tokens > --max-seq-len") - positions = target_positions(record) if split == "eval" else None - state["positions"] = None if positions is None else torch.as_tensor(positions, device="cuda") - if positions is not None: - state["targets"] = torch.as_tensor(np.asarray(ids, dtype=np.int64)[positions + 1], device="cuda") - with torch.inference_mode(): - model.forward(torch.as_tensor([ids], dtype=torch.long, device="cuda"), 0) - tokens_seen += len(ids) - if positions is not None: - logits_arrays = { - "positions": positions.astype(np.int32), - "target_token_ids": np.asarray(ids, dtype=np.int32)[positions + 1], - } - if rank == 0: - logits_arrays.update({k: np.concatenate(v) for k, v in state["captured"].items() if v}) - state["positions"] = state["targets"] = state["teacher"] = state["stats"] = None - if stats is not None: - stats.reduce(dist) - if rank == 0: - arrays = identity(session=name, split=split, records=records, config_sha=config_sha, chunk=args.max_seq_len) - if stats is not None: - stat_arrays = stats.arrays(layers) - if split == "eval": # REAP + routes only: per-split saliency, not ~0.9 GB of imatrix per sample - stat_arrays = {k: v for k, v in stat_arrays.items() if k.split("__")[-1] not in SPACES} - arrays.update(stat_arrays) - arrays.update(logits_arrays) - if student: + progress = args.out / f"progress-{args.sweep}-{shard}of{n_shards}.jsonl" + for chunk in chunk_sessions(todo, args.chunk_tokens, len(streams)): + if deadline is not None and time.time() > deadline: + print("deadline reached; stopping cleanly", flush=True) + break + t0 = time.time() + results = run_chunk(streamer, state, chunk, split=args.sweep, students=students, dims=dims, device="cuda") + for name, records in chunk: + ident = identity(session=name, split=args.sweep, records=records, config_sha=config_sha, chunk=args.max_seq_len) + for stream_name in streams: + arrays = ident | results[(stream_name, name)] + if stream_name != TEACHER: arrays["record_type"] = np.asarray("dsv4_masked_student_eval_v1") - arrays["survivors_sha256"] = np.asarray(survivors_sha) + arrays["survivors_sha256"] = np.asarray(survivors_sha[stream_name]) arrays["teacher_run_config_sha256"] = np.asarray(teacher_config_sha) - save_npz(target, arrays) - entry = { - "session": name, - "split": split, - "tokens": tokens_seen, - "seconds": round(time.time() - t0, 3), - "tok_per_s": round(tokens_seen / max(time.time() - t0, 1e-9), 1), - "elapsed_h": round((time.time() - started) / 3600, 3), - } - with progress.open("a") as handle: - handle.write(json.dumps(entry) + "\n") - print(json.dumps(entry), flush=True) - del stats - torch.cuda.empty_cache() - return True - - passes: list[tuple[Path, Path | None]] = [(args.out, None)] - for item in args.survivors: - keep, _, table = item.partition("=") - passes.append((args.out / f"student-{int(keep)}", Path(table))) - for out_dir, table in passes: - if rank == 0: - print(f"pass -> {out_dir}", flush=True) - if not run_pass(out_dir, mask_router(table)): - break - if world_size > 1: - dist.barrier() - dist.destroy_process_group() + save_npz(target(stream_name, name), arrays) + tokens = sum(len(r["token_ids"]) for _, records in chunk for r in records) + entry = { + "sessions": [name for name, _ in chunk], + "tokens": tokens, + "streams": len(streams), + "seconds": round(time.time() - t0, 3), + "tok_per_s": round(tokens * len(streams) / max(time.time() - t0, 1e-9), 1), + "max_gpu_gb": round(torch.cuda.max_memory_allocated() / 2**30, 1), + "elapsed_h": round((time.time() - started) / 3600, 3), + } + with progress.open("a") as handle: + handle.write(json.dumps(entry) + "\n") + print(json.dumps(entry), flush=True) + del results + torch.cuda.empty_cache() return 0 diff --git a/scripts/runpod/v41_reap/pod.sh b/scripts/runpod/v41_reap/pod.sh index f98ec30..7d0ec53 100755 --- a/scripts/runpod/v41_reap/pod.sh +++ b/scripts/runpod/v41_reap/pod.sh @@ -4,9 +4,10 @@ # # pod.sh fetch CPU pod: download the pinned checkpoint, verify every LFS sha256 # pod.sh pack CPU pod: build the calibration + eval pack from public datasets -# pod.sh gpu 8xH200 pod: convert, teacher pass, candidate scoring; stops itself +# pod.sh gpu GPU pod (1-8 GPUs, 80 GB+ each): calibration sweep, reap-select, +# eval sweep (teacher + every candidate side by side), scores # pod.sh prune KEEP CPU pod: write the pruned HF checkpoint for the Mac to pull -# pod.sh all one 8xH200 pod does everything: fetch || pack, gpu, prune $PRUNE +# pod.sh all one GPU pod does everything: fetch || pack, gpu, prune $PRUNE # # Everything lives under /workspace (a network volume, or the pod's own volume # disk in single-pod mode). @@ -17,11 +18,10 @@ REVISION=1bae0e1088993b9b82de08f59de5be5a34e42170 W=${WORKSPACE:-/workspace} KIT=$W/keep/scripts/runpod/v41_reap HF=$W/v41-hf -TP8=$W/v41-tp8 OUT=$W/out CANDIDATES=${CANDIDATES:-"256 192 128 96"} PRUNE=${PRUNE:-"192 96"} # REAP 50% and REAP 75% of 384 experts -MAX_GPU_HOURS=${MAX_GPU_HOURS:-6} # hard stop for the GPU pod (8xH200 secure ~ $42/h) +MAX_GPU_HOURS=${MAX_GPU_HOURS:-8} # hard stop for the GPU pod (1xH100 ~ $2.69/h, 1xH200 $3.59-5.29/h) export PYTHONPATH=$W/keep/src HF_HUB_ENABLE_HF_TRANSFER=1 watchdog() { # whatever happens next, the pod stops at MAX_GPU_HOURS; a failed stage stops it at once @@ -42,7 +42,7 @@ stop_pod() { fi } -fetch_small() { # tokenizer, configs, encoding/ and inference/: all pack and convert need besides weights +fetch_small() { # tokenizer, configs, encoding/ and inference/: all pack and collect need besides weights pip install -q "huggingface_hub[hf_transfer]>=0.34" "datasets>=3" tokenizers hf download "$MODEL_ID" --revision "$REVISION" --local-dir "$HF" --include "*.json" "encoding/*" "inference/*" } @@ -79,24 +79,32 @@ pack() { log "pack built: $(python3 -c "import json;m=json.load(open('$W/pack/manifest.json'));print(m['records'],'records',m['sessions'],'sessions')")" } +sweep() { # one collect.py per GPU, sessions sharded across them; extra args go to collect.py + local name=$1; shift + local n i pids=() + n=$(nvidia-smi -L | wc -l) + log "sweep $name on $n GPU(s)" + for ((i = 0; i < n; i++)); do + CUDA_VISIBLE_DEVICES=$i python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/pack/v41-pack.jsonl" \ + --out "$OUT" --sweep "$name" --shard "$i/$n" --deadline-utc "$deadline" "$@" \ + > "$W/sweep-$name-$i.log" 2>&1 & + pids+=($!) + done + for i in "${!pids[@]}"; do + wait "${pids[$i]}" || { log "sweep $name shard $i failed: see $W/sweep-$name-$i.log"; return 1; } + done +} + gpu() { local deadline deadline=$(date -u -d "+$((MAX_GPU_HOURS * 60 - 20)) minutes" +%FT%TZ) watchdog - cd "$HF/inference" - pip install -q -r requirements.txt + pip install -q -r "$HF/inference/requirements.txt" pip freeze > "$W/pip-freeze-gpu.txt" nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$W/pod.log" - if [[ ! -f "$TP8/model7-mp8.safetensors" ]]; then - log "convert -> $TP8" - python3 convert.py --hf-ckpt-path "$HF" --save-path "$TP8" --model-parallel 8 --expert-dtype fp4 - fi - - log "teacher pass" - torchrun --nproc-per-node 8 "$KIT/collect.py" --ckpt-path "$TP8" --config config.json \ - --pack "$W/pack/v41-pack.jsonl" --out "$OUT" --deadline-utc "$deadline" + sweep calibration log "finalize-reap + candidates: $CANDIDATES" python3 "$W/keep/benchmarks/produce_dsv4_teacher_cache.py" finalize-reap --out-dir "$OUT/calibration" @@ -108,9 +116,7 @@ gpu() { survivors+=(--survivors "$keep=$OUT/survivors-$keep.json") done - log "student passes" - torchrun --nproc-per-node 8 "$KIT/collect.py" --ckpt-path "$TP8" --config config.json \ - --pack "$W/pack/v41-pack.jsonl" --out "$OUT" --deadline-utc "$deadline" "${survivors[@]}" + sweep eval "${survivors[@]}" local students=() for keep in $CANDIDATES; do students+=(--student "$keep=$OUT/student-$keep"); done @@ -134,7 +140,6 @@ all() { pack > "$W/pack.log" 2>&1 wait "$fetch_pid" || { log "fetch failed: see $W/fetch.log"; return 1; } gpu - rm -rf "$TP8" # 510 GB; the pruned checkpoints come from the HF source for keep in $PRUNE; do prune "$keep"; done log "all done: $(echo "$PRUNE" | sed "s|[0-9]*|$W/v41-reap&|g"). Pod left running for the Mac to pull; watchdog still armed." } diff --git a/scripts/runpod/v41_reap/stream.py b/scripts/runpod/v41_reap/stream.py new file mode 100644 index 0000000..0011efe --- /dev/null +++ b/scripts/runpod/v41_reap/stream.py @@ -0,0 +1,333 @@ +"""DeepSeek-V4.1-Flash forward on one GPU, one layer at a time. + +Design: ``docs/deepseek-v4-flash/2026-11-06-v41-reap-single-gpu-design.md``. + +The reference ``Block`` modules run unchanged. Only the order of work changes: +for a chunk of records, each layer's weights are loaded once (straight from the +HF shards, no ``convert.py``), every record runs through that layer's attention, +then the MoE runs over whole groups of records at once. Between layers each +record keeps three things: + +- its hyper-connection residual; +- its ``pre_mix``; +- its slice of ``shared_attn``, the module-level slot the reference uses to hand + compressed KV, index keys, top-k indices and candidate blocks down the stack. + +Engram rows are gathered from the shard files on the host and never put on the +GPU. ``tests/test_runpod_v41_stream.py`` checks the whole thing against +``Transformer.forward`` on a toy model. +""" + +from __future__ import annotations + +import importlib +import json +import struct +import sys +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +INDEX = "model.safetensors.index.json" +SHARED_FIELDS = ("compress_kv", "index_k", "topk_idxs", "candidates") +ST_DTYPES = { + "F8_E4M3": torch.float8_e4m3fn, + "F8_E8M0": torch.float8_e8m0fnu, + "F32": torch.float32, + "BF16": torch.bfloat16, +} + + +def import_reference(ref_dir: Path, kernel: Any | None = None) -> Any: + """Import the reference ``model`` module, optionally with a stand-in ``kernel``.""" + + if kernel is not None: + sys.modules["kernel"] = kernel + path = str(ref_dir) + if path not in sys.path: + sys.path.insert(0, path) + return importlib.import_module("model") + + +# --------------------------------------------------------------------------- +# Weights +# --------------------------------------------------------------------------- + + +def reference_name(name: str) -> str: + """HF tensor name -> reference parameter name (``convert.py`` at --model-parallel 1).""" + + name = name.removeprefix("model.") + name = name.replace("self_attn", "attn") + if not name.startswith("vision."): + name = name.replace("mlp", "ffn") + return name.replace("weight_scale_inv", "scale").replace("e_score_correction_bias", "bias") + + +def read_header(path: Path) -> tuple[dict[str, Any], int]: + with path.open("rb") as handle: + (size,) = struct.unpack(" dict[str, torch.Tensor]: + """Every tensor under ``prefix``, keyed relative to it, with convert.py's fixups.""" + + from safetensors import safe_open + + wanted: dict[str, list[tuple[str, str]]] = {} + for ref_name, (hf_name, shard) in self.shard_of.items(): + if ref_name.startswith(prefix) and not any(s in ref_name for s in skip): + wanted.setdefault(shard, []).append((ref_name, hf_name)) + out: dict[str, torch.Tensor] = {} + for shard, names in wanted.items(): + with safe_open(str(self.root / shard), framework="pt", device="cpu") as handle: + for ref_name, hf_name in names: + out[ref_name[len(prefix):]] = handle.get_tensor(hf_name) + return fix_tensors(out) + + +def fix_tensors(sd: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """``convert.py``'s per-tensor changes: wo_a dequantized to bf16, FP4 experts viewed as fp4.""" + + for name in [n for n in sd if n.endswith("wo_a.weight")]: + scale_name = name.replace("weight", "scale") + if scale_name in sd: + weight, scale = sd[name], sd.pop(scale_name) + ob, ib = weight.size(0) // scale.size(0), weight.size(1) // scale.size(1) + assert (ob, ib) in ((32, 32), (128, 128)), (name, weight.shape, scale.shape) + full = weight.unflatten(0, (-1, ob)).unflatten(-1, (-1, ib)).float() * scale[:, None, :, None].float() + sd[name] = full.flatten(2, 3).flatten(0, 1).bfloat16() + for name, tensor in sd.items(): + if "experts" in name and tensor.dtype == torch.int8: + sd[name] = tensor.view(torch.float4_e2m1fn_x2) + return sd + + +# --------------------------------------------------------------------------- +# Engram rows on the host +# --------------------------------------------------------------------------- + + +class EngramRows: + """One layer's n-gram table as host byte arrays (np.memmap of the shard, or plain arrays).""" + + def __init__(self, weight: np.ndarray, scale: np.ndarray, scale_dtype: torch.dtype, block_size: int): + self.weight, self.scale = weight, scale # uint8 [rows, dim], uint8 [rows, scale bytes] + self.scale_dtype, self.block_size = scale_dtype, block_size + + @classmethod + def from_checkpoint(cls, ckpt: HFCheckpoint, layer: int, block_size: int) -> EngramRows: + arrays = {} + for leaf in ("weight", "scale"): + hf_name, shard = ckpt.shard_of[f"layers.{layer}.engram.embed.{leaf}"] + header, base = read_header(ckpt.root / shard) + info = header[hf_name] + start, end = info["data_offsets"] + rows = info["shape"][0] + arrays[leaf] = (np.memmap(ckpt.root / shard, dtype=np.uint8, mode="r", offset=base + start, + shape=(rows, (end - start) // rows)), info["dtype"]) + assert arrays["weight"][1] == "F8_E4M3", arrays["weight"][1] + return cls(arrays["weight"][0], arrays["scale"][0], ST_DTYPES[arrays["scale"][1]], block_size) + + def lookup(self, indices: torch.Tensor) -> torch.Tensor: + """``ParallelEngramEmbedding.forward`` at world_size 1, reading rows from the host.""" + + flat = indices.reshape(-1).cpu().numpy() + unique, inverse = np.unique(flat, return_inverse=True) # sorted: near-sequential reads + w = torch.from_numpy(np.ascontiguousarray(self.weight[unique]))[torch.from_numpy(inverse)] + s = torch.from_numpy(np.ascontiguousarray(self.scale[unique]))[torch.from_numpy(inverse)] + w = w.to(indices.device).view(torch.float8_e4m3fn).view(*indices.shape, -1) + s = s.to(indices.device).view(self.scale_dtype).view(*indices.shape, -1) + values = w.float().unflatten(-1, (-1, self.block_size)) * s.float().unsqueeze(-1) + return values.flatten(-2).to(torch.bfloat16) + + +class HostTableEmbedding(torch.nn.Module): + """Stand-in for ``ParallelEngramEmbedding``: same forward, no table on the device.""" + + def __init__(self, num_embeddings: int, dim: int): + super().__init__() + self.num_embeddings, self.dim = num_embeddings, dim + self.rows: EngramRows | None = None + + def forward(self, indices: torch.Tensor) -> torch.Tensor: + assert self.rows is not None, "engram rows not attached" + return self.rows.lookup(indices) + + +# --------------------------------------------------------------------------- +# shared_attn per record +# --------------------------------------------------------------------------- + + +def pack_bits(mask: torch.Tensor) -> tuple[torch.Tensor, int]: + n = mask.shape[-1] + padded = torch.nn.functional.pad(mask.to(torch.uint8), (0, -n % 8)).unflatten(-1, (-1, 8)) + shifts = torch.arange(8, dtype=torch.uint8, device=mask.device) + return (padded << shifts).sum(dim=-1, dtype=torch.uint8), n + + +def unpack_bits(packed: tuple[torch.Tensor, int]) -> torch.Tensor: + bits, n = packed + shifts = torch.arange(8, dtype=torch.uint8, device=bits.device) + return ((bits.unsqueeze(-1) >> shifts) & 1).bool().flatten(-2)[..., :n] + + +# --------------------------------------------------------------------------- +# The streamer +# --------------------------------------------------------------------------- + + +@dataclass +class Item: + """One record in one stream (the teacher, or a masked-router student).""" + + tokens: torch.Tensor # [1, s] long + group: Any # items sharing a group go through the MoE together + h: torch.Tensor | None = None + pre_mix: torch.Tensor | None = None + hashes: torch.Tensor | None = None + shared: dict[str, Any] = field(default_factory=dict) + restored: dict[str, Any] = field(default_factory=dict) + scratch: dict[str, Any] = field(default_factory=dict) + + +class LayerStreamer: + def __init__( + self, + ref: Any, + args: Any, + load_layer: Callable[[str], dict[str, torch.Tensor]], + engram_rows: dict[int, EngramRows], + hash_state: Any, + device: str, + ): + self.ref, self.args, self.device = ref, args, device + self.load_layer, self.engram_rows, self.hash_state = load_layer, engram_rows, hash_state + ref.world_size, ref.rank = 1, 0 + ref.default_dtype = torch.float8_e4m3fn if args.dtype == "fp8" else torch.bfloat16 + self.layout = ref.EngramLayout.from_args(args) + self.hc = args.hc_mult + with torch.device(device): + self.embed = ref.ParallelEmbedding(args.vocab_size, args.dim) + self.norm = ref.RMSNorm(args.dim, args.norm_eps) + self.head = ref.ParallelHead(args.vocab_size, args.dim, args.norm_eps, args.hc_eps) + for prefix, module in (("embed.", self.embed), ("norm.", self.norm), ("head.", self.head)): + module.load_state_dict(load_layer(prefix), strict=True) + self._pool = ThreadPoolExecutor(max_workers=1) + + def build_block(self, layer: int, state: dict[str, torch.Tensor]) -> Any: + ref = self.ref + real = ref.ParallelEngramEmbedding + ref.ParallelEngramEmbedding = HostTableEmbedding + try: + with torch.device(self.device): + block = ref.Block(layer, self.args, self.layout) + finally: + ref.ParallelEngramEmbedding = real + block.load_state_dict(state, strict=True) + if block.engram is not None: + block.engram.embed.rows = self.engram_rows[layer] + return block + + # -- per-record shared_attn ------------------------------------------------ + + def restore(self, item: Item) -> None: + shared = self.ref.shared_attn + item.restored = {} + for name in SHARED_FIELDS: + value = item.shared.get(name) + if name == "candidates" and value is not None: + value = unpack_bits(value) + item.restored[name] = value + setattr(shared, name, value) + + def snapshot(self, item: Item, block: Any) -> None: + shared = self.ref.shared_attn + own = {id(b) for b in block.buffers()} + for name in SHARED_FIELDS: + value = getattr(shared, name) + if value is None or value is item.restored.get(name): + continue # unchanged by this layer: keep the stored copy + if id(value) in own: # a view of this block's cache: the next record overwrites it + value = value.clone() + item.shared[name] = pack_bits(value) if name == "candidates" else value + for name in SHARED_FIELDS: + setattr(shared, name, None) + + # -- the forward ------------------------------------------------------------- + + def start(self, items: list[Item]) -> None: + for item in items: + h = self.embed(item.tokens) + item.h = h.unsqueeze(2).repeat(1, 1, self.hc, 1) + item.pre_mix = self.ref.make_identity_pre_mix(item.h, self.hc) + if self.hash_state is not None: + item.hashes = self.hash_state(item.tokens, 0) + + def run_layer(self, block: Any, items: list[Item], before_moe: Callable[[int, Any, Any], None] | None) -> None: + for item in items: # attention: one record at a time, whole sequence + if block.engram is not None: + item.h = block.engram(item.h, item.hashes[:, :, block.engram.layer_hash_index, :], None) + self.restore(item) + residual = item.h + attn_pre, attn_post, attn_comb = block.hc_mixes(item.h, block.hc_attn_fn, block.hc_attn_scale, block.hc_attn_base) + x = block.attn_norm(block.hc_pre(item.h, item.pre_mix)) + x = block.attn(x, 0) + item.h = block.hc_post(x, residual, attn_post, attn_comb) + item.scratch["attn_pre"] = attn_pre + self.snapshot(item, block) + + groups: dict[Any, list[Item]] = {} + for item in items: + groups.setdefault(item.group, []).append(item) + for key, members in groups.items(): # FFN: the whole group through the MoE at once + xs = [] + for item in members: + ffn_pre, ffn_post, ffn_comb = block.hc_mixes(item.h, block.hc_ffn_fn, block.hc_ffn_scale, block.hc_ffn_base) + xs.append(block.ffn_norm(block.hc_pre(item.h, item.scratch.pop("attn_pre")))) + item.scratch["ffn"] = (ffn_pre, ffn_post, ffn_comb) + if before_moe is not None: + before_moe(block.layer_id, block, key) + y = block.ffn(torch.cat(xs, dim=1) if len(xs) > 1 else xs[0], None) + for item, part in zip(members, y.split([x.size(1) for x in xs], dim=1)): + ffn_pre, ffn_post, ffn_comb = item.scratch.pop("ffn") + item.h = block.hc_post(part, item.h, ffn_post, ffn_comb) + item.pre_mix = ffn_pre + + def forward(self, items: list[Item], before_moe: Callable[[int, Any, Any], None] | None = None) -> None: + """Run ``items`` through every backbone layer; leaves each ``item.h`` / ``item.pre_mix``.""" + + n_layers = int(self.args.n_layers) + skip = (".engram.embed.",) + pending = self._pool.submit(self.load_layer, "layers.0.", skip) + self.start(items) + for layer in range(n_layers): + state = pending.result() + if layer + 1 < n_layers: + pending = self._pool.submit(self.load_layer, f"layers.{layer + 1}.", skip) + block = self.build_block(layer, state) + del state + self.run_layer(block, items, before_moe) + del block + for item in items: + item.shared.clear() + + def final_hidden(self, item: Item) -> torch.Tensor: + """``self.norm(layer.hc_pre(h, pre_mix))``: the head's input, [1, s, dim].""" + + return self.norm(self.ref.Block.hc_pre(None, item.h, item.pre_mix)) diff --git a/scripts/runpod/v41_reap/torch_kernel.py b/scripts/runpod/v41_reap/torch_kernel.py new file mode 100644 index 0000000..23cec13 --- /dev/null +++ b/scripts/runpod/v41_reap/torch_kernel.py @@ -0,0 +1,62 @@ +"""Plain-torch stand-ins for the reference ``inference/kernel.py`` (tilelang, GPU only). + +Used by the CPU tests to run DeepSeek's real ``model.py`` at toy size: install it +as ``sys.modules["kernel"]`` before importing ``model``. It covers the bf16 path +only. The quantizers are no-ops (they fake-quantize in place on the GPU) and the +quantized GEMMs refuse to run, so a test model must use ``dtype="bf16"`` and +``expert_dtype=None``. ``sparse_attn`` and ``hc_split_sinkhorn`` are exact ports +of the kernels' math. +""" + +from __future__ import annotations + +import torch + + +def act_quant(x, block_size=128, scale_fmt=None, scale_dtype=torch.float32, inplace=False): + if inplace: + return x + raise NotImplementedError("fp8 activations: CPU tests use bf16 weights") + + +def fp4_act_quant(x, block_size=32, inplace=False, scale_dtype=None): + if inplace: + return x + raise NotImplementedError("fp4 activations: CPU tests use bf16 weights") + + +def fp8_gemm(*args, **kwargs): + raise NotImplementedError("fp8 GEMM: CPU tests use bf16 weights") + + +def fp4_gemm(*args, **kwargs): + raise NotImplementedError("fp4 GEMM: CPU tests use bf16 weights") + + +def sparse_attn(q, kv, attn_sink, topk_idxs, softmax_scale): + """q [b,m,h,d], kv [b,n,d] (one shared head), topk_idxs [b,m,k] with -1 = empty slot. + Softmax over the gathered slots plus a per-head sink logit; a row with no valid slot is 0.""" + + b = q.shape[0] + valid = topk_idxs >= 0 + gathered = kv.float()[torch.arange(b)[:, None, None], topk_idxs.clamp_min(0).long()] # [b,m,k,d] + scores = torch.einsum("bmhd,bmkd->bmhk", q.float(), gathered) * softmax_scale + scores = scores.masked_fill(~valid[:, :, None, :], -torch.inf) + row_max = scores.amax(dim=-1, keepdim=True).clamp_min(-1e30) # the kernel's finite floor + p = torch.exp(scores - row_max) + denom = p.sum(dim=-1) + torch.exp(attn_sink.float()[None, None, :] - row_max[..., 0]) + o = torch.einsum("bmhk,bmkd->bmhd", p, gathered) / denom[..., None] + return o.to(q.dtype) + + +def hc_split_sinkhorn(mixes, hc_scale, hc_base, hc_mult=4, sinkhorn_iters=20, eps=1e-6): + hc = hc_mult + pre = torch.sigmoid(mixes[..., :hc] * hc_scale[0] + hc_base[:hc]) + eps + post = 2 * torch.sigmoid(mixes[..., hc : 2 * hc] * hc_scale[1] + hc_base[hc : 2 * hc]) + comb = (mixes[..., 2 * hc :] * hc_scale[2] + hc_base[2 * hc :]).unflatten(-1, (hc, hc)) + comb = comb.softmax(dim=-1) + eps + comb = comb / (comb.sum(dim=-2, keepdim=True) + eps) + for _ in range(sinkhorn_iters - 1): + comb = comb / (comb.sum(dim=-1, keepdim=True) + eps) + comb = comb / (comb.sum(dim=-2, keepdim=True) + eps) + return pre, post, comb diff --git a/tests/fixtures/v41_reference/LICENSE b/tests/fixtures/v41_reference/LICENSE new file mode 100644 index 0000000..d62e3be --- /dev/null +++ b/tests/fixtures/v41_reference/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 DeepSeek + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/tests/fixtures/v41_reference/README.md b/tests/fixtures/v41_reference/README.md new file mode 100644 index 0000000..0cb4f7c --- /dev/null +++ b/tests/fixtures/v41_reference/README.md @@ -0,0 +1,3 @@ +`model.py` and `engram.py` are copied unchanged from `deepseek-ai/DeepSeek-V4.1-Flash-1017`, in the `inference/` directory at revision `1bae0e1088993b9b82de08f59de5be5a34e42170`. They are MIT-licensed; the license is in `LICENSE`. `model.py` has sha256 `e5decd76fe70fd56ef2f10124d245c6f64fb6a0003cc9068a1d3182a168b8d2d`. + +The CPU tests in `tests/test_runpod_v41_stream.py` run them at toy size. They swap in `scripts/runpod/v41_reap/torch_kernel.py` for the reference's tilelang `kernel.py`, and stub out the modules used only for vision. diff --git a/tests/fixtures/v41_reference/engram.py b/tests/fixtures/v41_reference/engram.py new file mode 100644 index 0000000..a36bc6c --- /dev/null +++ b/tests/fixtures/v41_reference/engram.py @@ -0,0 +1,184 @@ +from dataclasses import dataclass + +import numpy as np +import torch +from sympy import isprime +from torch import nn + + +def find_next_prime(start: int, seen_primes: set[int]) -> int: + """The smallest prime above `start` that has not been handed out yet.""" + candidate = start + 1 + while not isprime(candidate) or candidate in seen_primes: + candidate += 1 + return candidate + + +def build_compressed_token_map(tokenizer) -> tuple[list[int], int]: + """Map every token id onto a smaller id space where tokens that normalize alike collapse together. + + N-grams are hashed over these compressed ids, so " The", "the" and "THE" all hash the same way. + Returns the lookup plus the size of the compressed vocab -- and that size matters beyond bounds + checking, because every hash multiplier is derived from it. + """ + from tokenizers import Regex, normalizers + + # a private-use char, so a token that is exactly one space survives Strip() instead of + # collapsing to the empty string and merging with unrelated tokens + sentinel = "\ue000" + normalizer = normalizers.Sequence( + [ + normalizers.NFKC(), + normalizers.NFD(), + normalizers.StripAccents(), + normalizers.Lowercase(), + normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), + normalizers.Replace(Regex(r"^ $"), sentinel), + normalizers.Strip(), + normalizers.Replace(sentinel, " "), + ] + ) + + # the raw Rust tokenizer, matching what training decodes with (no clean_up_tokenization_spaces) + backend = tokenizer.backend_tokenizer + key_to_new: dict[str, int] = {} + lookup = [0] * len(tokenizer) + for token_id in range(len(tokenizer)): + text = backend.decode([token_id], skip_special_tokens=False) + if "\ufffd" in text: + # a partial UTF-8 byte token: nothing to normalize, so key it by its raw form + key = backend.id_to_token(token_id) + else: + normalized = normalizer.normalize_str(text) + key = normalized if normalized else text + + new_id = key_to_new.get(key) + if new_id is None: + new_id = len(key_to_new) + key_to_new[key] = new_id + lookup[token_id] = new_id + + return lookup, len(key_to_new) + + +def compute_hash_multipliers( + layer_ids: tuple[int, ...], max_ngram_size: int, tokenizer_vocab_size: int +) -> torch.Tensor: + """One multiplier per (layer, lookback), from a per-layer RNG so layers hash differently. + + Kept odd, and bounded so that `token_id * multiplier` cannot overflow int64. + """ + max_long = np.iinfo(np.int64).max + multiplier_bound = max(1, (max_long // tokenizer_vocab_size) // 2) + rows = [] + for layer_id in layer_ids: + generator = np.random.default_rng(10007 * layer_id) + values = generator.integers( + low=0, + high=multiplier_bound, + size=(max_ngram_size,), + dtype=np.int64, + ) + rows.append(torch.tensor(values * 2 + 1)) + return torch.stack(rows) + + +@dataclass(frozen=True) +class EngramLayout: + """Bucket layout of the n-gram hash tables. + + A position is hashed as `max_ngram_size - 1` n-grams (2-gram .. max_ngram_size-gram), each split + over `n_heads` heads. Every (n-gram size, head) pair owns its own prime-sized bucket range in the + layer's table; the primes are drawn in order and never reused, which keeps the ranges disjoint. + """ + + max_ngram_size: int + layer_ids: tuple[int, ...] + num_embeddings: tuple[int, ...] # table rows, per engram layer + primes: tuple[tuple[tuple[int, ...], ...], ...] # [layer][n-gram size][head] bucket modulus + n_heads: int + head_dim: int + + @classmethod + def from_args(cls, args) -> "EngramLayout | None": + layer_ids = tuple(args.engram_layer_ids) + if not layer_ids: + return None + max_ngram_size, n_heads = args.engram_max_ngram_size, args.engram_n_heads + primes, seen = [], set() + for _ in layer_ids: + per_ngram = [] + for _ in range(max_ngram_size - 1): + sizes, current = [], args.engram_vocab_size - 1 + for _ in range(n_heads): + current = find_next_prime(current, seen) + seen.add(current) + sizes.append(current) + per_ngram.append(tuple(sizes)) + primes.append(tuple(per_ngram)) + return cls( + max_ngram_size=max_ngram_size, + layer_ids=layer_ids, + num_embeddings=tuple(args.engram_num_embeddings), + primes=tuple(primes), + n_heads=n_heads, + head_dim=args.engram_head_dim, + ) + + +class NgramHashState(nn.Module): + """Maps each position to the hash ids of the n-grams ending there. + + Ids go through the compressed table, then each position is hashed with the `max_ngram_size - 1` + tokens before it. Look-back stops at the start of the sequence and at any dead token (an image + span, cached as DEAD), so an n-gram never spans one. The cache carries all of this across the + prefill/decode split. + """ + + DEAD = -1 + + def __init__(self, args, layout: EngramLayout, tokenizer): + super().__init__() + self.layout = layout + # every hash multiplier derives from the compressed vocab size, so a mismatch there would + # silently rehash the whole table + token_map, vocab_size = build_compressed_token_map(tokenizer) + assert vocab_size == args.engram_compressed_vocab_size, (vocab_size, args.engram_compressed_vocab_size) + self.pad_id = token_map[args.engram_pad_id] + flat = [[p for per_ngram in layer for p in per_ngram] for layer in layout.primes] + offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat] + multipliers = compute_hash_multipliers(layout.layer_ids, layout.max_ngram_size, vocab_size) + self.register_buffer("primes", torch.tensor(layout.primes), persistent=False) + self.register_buffer("offsets", torch.tensor(np.array(offsets)), persistent=False) + self.register_buffer("multipliers", multipliers, persistent=False) + self.register_buffer("token_map", torch.tensor(token_map), persistent=False) + self.register_buffer( + "cache", torch.empty(args.max_batch_size, args.max_seq_len, dtype=torch.int64), persistent=False + ) + + @torch.inference_mode() + def forward(self, input_ids: torch.Tensor, start_pos: int, token_mask: torch.Tensor | None = None) -> torch.Tensor: + """token_mask: [B, L], False for tokens that take no part in an n-gram (image spans). + Returns the hash ids, shaped [B, L, n_engram_layers, n_hash_cols].""" + batch, seqlen = input_ids.shape + compressed = self.token_map[input_ids] + if token_mask is not None: + compressed = torch.where(token_mask, compressed, self.DEAD) + self.cache[:batch, start_pos : start_pos + seqlen] = compressed + + positions = torch.arange(start_pos, start_pos + seqlen, device=input_ids.device).expand(batch, seqlen) + tokens, blocked = [], torch.zeros_like(positions, dtype=torch.bool) + for shift in range(self.layout.max_ngram_size): + source = self.cache[:batch].gather(1, (positions - shift).clamp_min(0)) + blocked = blocked | (positions < shift) | (source == self.DEAD) + tokens.append(torch.where(blocked, self.pad_id, source)) + tokens = torch.stack(tokens, dim=-1) # [B, L, max_ngram_size] + + # XOR the multiplied ids together one lookback at a time, so the running value after step i + # is the hash of the (i+1)-gram; each lands in its own prime-sized bucket range + products = tokens.unsqueeze(2) * self.multipliers # [B, L, n_engram_layers, max_ngram_size] + rolling, hashes = products[..., 0], [] + for i in range(1, self.layout.max_ngram_size): + rolling = torch.bitwise_xor(rolling, products[..., i]) + hashes.append(rolling.unsqueeze(-1) % self.primes[:, i - 1]) + return torch.cat(hashes, dim=-1) + self.offsets diff --git a/tests/fixtures/v41_reference/model.py b/tests/fixtures/v41_reference/model.py new file mode 100644 index 0000000..46cc831 --- /dev/null +++ b/tests/fixtures/v41_reference/model.py @@ -0,0 +1,1312 @@ +import math +from contextlib import contextmanager +from dataclasses import dataclass +from functools import lru_cache +from typing import Literal + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from engram import EngramLayout, NgramHashState +from image_processor import IMAGE, IMAGE_END, IMAGE_NEW_LINE, IMAGE_START +from kernel import ( + act_quant, + fp4_act_quant, + fp4_gemm, + fp8_gemm, + hc_split_sinkhorn, + sparse_attn, +) +from torch import nn +from vision import Aligner, ViT + +# Set once by Transformer.__init__; one model per process, so layers just read them. +world_size = 1 +rank = 0 +default_dtype = torch.float8_e4m3fn # storage dtype for Linear weights, from ModelArgs.dtype +fp8_block_size = 32 # one fp8 scale per 32x32 weight block / 32 activations +fp4_block_size = 32 # one fp4 scale per 32 elements along K +scale_fmt = "ue8m0" +scale_dtype = torch.float8_e8m0fnu + + +@contextmanager +def set_dtype(dtype): + """Temporarily override torch's default dtype, restoring it even if the body raises.""" + prev = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(prev) + + +@dataclass +class ModelArgs: + """Field names are exactly the config JSON keys. The defaults are a small model that + `python model.py` can run, not the released shapes -- though the scale-independent + values (norm_eps, score_func, hc_*, engram_*) do match it.""" + + # runtime limits rather than model shape: they size the KV caches + max_batch_size: int = 4 + max_seq_len: int = 4096 + temperature: float = 1 + dtype: Literal["bf16", "fp8"] = "fp8" + expert_dtype: Literal["fp4"] | None = "fp4" + vocab_size: int = 129280 + dim: int = 1024 + moe_inter_dim: int = 1024 + n_layers: int = 5 + n_mtp_layers: int = 1 # extra draft layers appended after the backbone, indices n_layers.. + n_heads: int = 16 + # moe + n_routed_experts: int = 8 + n_shared_experts: int = 1 + n_activated_experts: int = 2 + score_func: Literal["softmax", "sigmoid", "sqrtsoftplus"] = "sqrtsoftplus" + gate_temp: float = 1.0 + norm_topk_prob: bool = True + route_scale: float = 1.0 + swiglu_limit: float = 0.0 + # attention: latent q/kv projections, plus a LoRA-factorised output projection over o_groups + q_lora_rank: int = 256 + head_dim: int = 128 + rope_head_dim: int = 32 + norm_eps: float = 1e-20 + o_groups: int = 8 + o_lora_rank: int = 256 + # sparse attention: every layer attends over a sliding window, and may add compressed KV on top + window_size: int = 128 + # one entry per layer, MTP layers included: 0 = sliding window only, r = KV compressed r-to-1 + compress_ratios: tuple[int, ...] = (0, 2, 2, 1, 1, 0) + # layers sharing a ratio also share one compressed KV and one indexer, produced by the first + kv_source_layers: tuple[int, ...] = (1, 3) + index_source_layers: tuple[int, ...] = (1, 3) + # rope, with YaRN extrapolation when original_seq_len > 0. Compressed KV rotates at its own + # theta because one latent stands for compress_ratio tokens, so its positions are further apart. + compress_rope_theta: float = 40000.0 + original_seq_len: int = 0 + rope_theta: float = 10000.0 + rope_factor: float = 40 + beta_fast: int = 32 + beta_slow: int = 1 + # the indexer: a small extra attention that scores compressed positions, so each query can keep + # just `index_topk` of them. Names match DeepSeek-V3.2-Exp, where this mechanism first appeared. + index_n_heads: int = 16 + index_head_dim: int = 64 + index_topk: int = 64 + # candidate pre-filtering: candidate_source_layer < 0 turns it off and the other two are unused + candidate_source_layer: int = -1 + candidate_topk_blocks: int = 0 + candidate_block_size: int = 0 + # hyper-connections: the residual stream is carried as hc_mult parallel copies + hc_mult: int = 4 + hc_sinkhorn_iters: int = 20 + hc_eps: float = 1e-6 + # engram: n-gram hash lookups added into the residual stream at a few layers + engram_layer_ids: tuple[int, ...] = () + engram_num_embeddings: tuple[int, ...] = () # unpadded table rows; each rank allocates ceil(rows / world_size) + engram_max_ngram_size: int = 1 + engram_vocab_size: int = 0 # bucket size each (n-gram size, head) starts searching primes from + engram_n_heads: int = 0 + engram_head_dim: int = 0 + engram_pad_id: int = 2 # token that fills n-gram slots with no history; matches training + # size of the compressed tokenizer vocab; every hash multiplier is derived from it + engram_compressed_vocab_size: int = 0 + # vision (VL); vision_n_layers == 0 disables the vision path + vision_n_layers: int = 0 + vision_dim: int = 1024 + vision_n_heads: int = 16 + vision_inter_dim: int = 2816 + vision_patch_size: int = 14 + vision_rope_theta: float = 10000.0 + vision_downsample_ratio: int = 3 + vision_max_n_token: int = 1024 + vision_min_pixels: int = 544 * 544 + vision_max_wh_ratio: int | None = None + # raw id of <｜deepseek_image｜>; every position of an image span carries this id in input_ids + image_token_id: int = 129264 + # dspark draft head. Only the forward pass is implemented here -- nothing calls forward_spec, + # so these are read but the speculative-decoding loop itself is out of scope for this repo. + dspark_block_size: int = 0 + dspark_noise_token_id: int = 0 + dspark_target_layer_ids: tuple[int, ...] = () + dspark_markov_rank: int = 256 + dspark_n_routed_experts: int = 0 + dspark_n_activated_experts: int = 0 + + @property + def vision_enabled(self) -> bool: + return self.vision_n_layers > 0 + + def get_moe_config(self, layer_id: int) -> tuple[int, int]: + """Return the routed/activated expert counts for a given layer.""" + if layer_id < self.n_layers: + return self.n_routed_experts, self.n_activated_experts + return ( + self.dspark_n_routed_experts or self.n_routed_experts, + self.dspark_n_activated_experts or self.n_activated_experts, + ) + + +class ParallelEmbedding(nn.Module): + """Embedding sharded along the vocab dimension. Each rank holds vocab_size // world_size rows. + Out-of-range indices are zero-masked before all_reduce to combine partial embeddings.""" + + def __init__(self, vocab_size: int, dim: int): + super().__init__() + self.vocab_size = vocab_size + self.dim = dim + assert vocab_size % world_size == 0, ( + f"Vocabulary size must be divisible by world size (world_size={world_size})" + ) + self.part_vocab_size = vocab_size // world_size + self.vocab_start_idx = rank * self.part_vocab_size + self.vocab_end_idx = self.vocab_start_idx + self.part_vocab_size + self.weight = nn.Parameter(torch.empty(self.part_vocab_size, self.dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if world_size > 1: + # ids off this rank read row 0 then get zeroed, so the all_reduce sums one real row + mask = (x < self.vocab_start_idx) | (x >= self.vocab_end_idx) + x = x - self.vocab_start_idx + x[mask] = 0 + y = F.embedding(x, self.weight) + if world_size > 1: + y[mask] = 0 + dist.all_reduce(y) + return y + + +def linear(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: + """Pick a GEMM from the weight dtype. Quantized weights need a quantized activation, and both + fp4 and fp8 weights take an fp8 one -- for fp4 the kernel handles the mixed precision.""" + assert bias is None + + if weight.dtype == torch.float4_e2m1fn_x2: + x, s = act_quant(x, fp8_block_size, scale_fmt, scale_dtype) + return fp4_gemm( + x, + s, + weight, + weight.scale, + scale_dtype, + act_block_size=fp8_block_size, + ) + elif weight.dtype == torch.float8_e4m3fn: + x, s = act_quant(x, fp8_block_size, scale_fmt, scale_dtype) + return fp8_gemm( + x, + s, + weight, + weight.scale, + scale_dtype, + block_size=fp8_block_size, + ) + else: + return F.linear(x, weight) + + +class Linear(nn.Module): + """bf16, fp8 or fp4 weights. Quantized ones get a `scale`, also attached to `.weight` so that + `linear()` can reach it.""" + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype=None): + super().__init__() + self.in_features = in_features + self.out_features = out_features + dtype = dtype or default_dtype + if dtype == torch.float4_e2m1fn_x2: + # two values per byte: [out, in] logically, [out, in//2] stored + self.weight = nn.Parameter(torch.empty(out_features, in_features // 2, dtype=torch.float4_e2m1fn_x2)) + self.weight.scale = self.scale = nn.Parameter( + torch.empty(out_features, in_features // fp4_block_size, dtype=torch.float8_e8m0fnu) + ) + elif dtype == torch.float8_e4m3fn: + self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype)) + self.weight.scale = self.scale = nn.Parameter( + torch.empty( + (out_features + fp8_block_size - 1) // fp8_block_size, + (in_features + fp8_block_size - 1) // fp8_block_size, + dtype=torch.float8_e8m0fnu, + ) + ) + else: + self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype)) + self.register_parameter("scale", None) + if bias: + self.bias = nn.Parameter(torch.empty(out_features)) + else: + self.register_parameter("bias", None) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return linear(x, self.weight, self.bias) + + +class ColumnParallelLinear(Linear): + """Splits the output dim across ranks; each rank's slice of the output is already complete.""" + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype=None): + assert out_features % world_size == 0, ( + f"Output features must be divisible by world size (world_size={world_size})" + ) + self.part_out_features = out_features // world_size + super().__init__(in_features, self.part_out_features, bias, dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return linear(x, self.weight, self.bias) + + +class RowParallelLinear(Linear): + """Splits the reduction dim, so each rank holds a partial sum: hence the fp32 all_reduce, with + the bias added only after it.""" + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype=None): + assert in_features % world_size == 0, ( + f"Input features must be divisible by world size (world_size={world_size})" + ) + self.part_in_features = in_features // world_size + super().__init__(self.part_in_features, out_features, bias, dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = linear(x, self.weight, None) + if world_size > 1: + y = y.float() + dist.all_reduce(y) + if self.bias is not None: + y += self.bias + return y.type_as(x) + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x: torch.Tensor): + dtype = x.dtype + x = x.float() + var = x.square().mean(-1, keepdim=True) + x = x * torch.rsqrt(var + self.eps) + return (self.weight * x).to(dtype) + + +class ParallelEngramEmbedding(nn.Module): + """The n-gram hash table, sharded over its rows. Stays fp8: rows are dequantized on lookup.""" + + def __init__(self, num_embeddings: int, dim: int): + super().__init__() + self.num_embeddings = num_embeddings + self.dim = dim + self.part_num_embeddings = (num_embeddings + world_size - 1) // world_size + self.vocab_start_idx = rank * self.part_num_embeddings + self.vocab_end_idx = self.vocab_start_idx + self.part_num_embeddings + + self.block_size = fp8_block_size + # the table stays fp8 as stored: rows are dequantized with `scale` on lookup + self.weight = nn.Parameter(torch.empty(self.part_num_embeddings, dim, dtype=torch.float8_e4m3fn)) + self.scale = nn.Parameter(torch.empty(self.part_num_embeddings, dim // self.block_size, dtype=scale_dtype)) + + def forward(self, indices: torch.Tensor) -> torch.Tensor: + mask = (indices < self.vocab_start_idx) | (indices >= self.vocab_end_idx) + local_indices = indices - self.vocab_start_idx + local_indices = local_indices.masked_fill(mask, 0) + + values = F.embedding(local_indices, self.weight) + scales = F.embedding(local_indices, self.scale) + values = values.float().unflatten(-1, (-1, self.block_size)) * scales.float().unsqueeze(-1) + values = values.flatten(-2).to(torch.bfloat16) + values = values.masked_fill(mask.unsqueeze(-1), 0) + + if world_size > 1: + dist.all_reduce(values) + return values + + +class Engram(nn.Module): + """Writes an n-gram lookup into the residual stream, gated by how well it matches that stream. + + The hash ids fetch `n_hash_cols` rows; `wkv` turns them into one key per hc copy plus a shared + value. The gate is a normalized dot product of stream against key. + """ + + def __init__(self, args: ModelArgs, layer_id: int, layout: EngramLayout): + super().__init__() + self.layer_id = layer_id + self.layer_hash_index = layout.layer_ids.index(layer_id) + self.dim = args.dim + self.hc_mult = args.hc_mult + self.clamp_value = 1e-6 + + self.embed = ParallelEngramEmbedding(layout.num_embeddings[self.layer_hash_index], layout.head_dim) + n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads + self.wkv = Linear(n_hash_cols * layout.head_dim, args.dim * (args.hc_mult + 1)) + self.eps = args.norm_eps + self.q_weight = nn.Parameter(torch.ones(args.hc_mult, args.dim)) + self.k_weight = nn.Parameter(torch.ones(args.hc_mult, args.dim)) + + def forward(self, x: torch.Tensor, hash_ids: torch.Tensor, token_mask: torch.Tensor | None = None) -> torch.Tensor: + """x: [B, L, hc_mult, dim]; hash_ids: [B, L, n_hash_cols]; token_mask: [B, L], False shuts + the gate so those positions pass through untouched.""" + kv = self.wkv(self.embed(hash_ids).flatten(-2)) + key, value = kv.split([self.hc_mult * self.dim, self.dim], dim=-1) + key = key.float().unflatten(-1, (self.hc_mult, self.dim)) + weight = self.q_weight.float() * self.k_weight.float() # only ever used as a product + h, eps = x.float(), self.eps + # normalized per (token, hc copy) over `dim`, NOT jointly over the copies + rstd = torch.rsqrt(h.square().mean(-1) + eps) * torch.rsqrt(key.square().mean(-1) + eps) + dot = (h * weight * key).sum(-1) * rstd * self.dim**-0.5 + # signed sqrt before the sigmoid, matching the training kernel + gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(self.clamp_value).sqrt(), dot)) + if token_mask is not None: + gate = gate.masked_fill(~token_mask.unsqueeze(-1), 0) + return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype) + + +@lru_cache(2) +def precompute_freqs_cis(dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow) -> torch.Tensor: + """Rotary frequencies as complex exponentials, one row per position. + + With original_seq_len > 0 this applies YaRN: dimensions whose wavelength already fits inside the + training context keep their frequency, those far beyond it are divided by `factor`, and the + `beta_fast`..`beta_slow` band in between is faded across with a linear ramp. + """ + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + if original_seq_len > 0: + # the dim whose wavelength completes `rotations` turns over the training context + def corrected_dim(rotations): + return dim * math.log(original_seq_len / (rotations * 2 * math.pi)) / (2 * math.log(base)) + + low = max(math.floor(corrected_dim(beta_fast)), 0) + high = min(math.ceil(corrected_dim(beta_slow)), dim - 1) + ramp = ((torch.arange(dim // 2, dtype=torch.float32) - low) / max(high - low, 1e-3)).clamp(0, 1) + smooth = 1 - ramp + freqs = freqs / factor * (1 - smooth) + freqs * smooth + + freqs = torch.outer(torch.arange(seqlen), freqs) + return torch.polar(torch.ones_like(freqs), freqs) + + +def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False) -> torch.Tensor: + """Rotate `x` in place, taking adjacent element pairs as complex numbers. Accepts [b, s, d] and + [b, s, h, d]; `inverse` conjugates the rotation, which is how the attention output gets the + query's rotation removed again so the cache can stay in one shared rotated form.""" + y = x + x = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) + if inverse: + freqs_cis = freqs_cis.conj() + if x.ndim == 3: + freqs_cis = freqs_cis.view(1, x.size(1), x.size(-1)) + else: + freqs_cis = freqs_cis.view(1, x.size(1), 1, x.size(-1)) + x = torch.view_as_real(x * freqs_cis).flatten(-2) + y.copy_(x) + return y + + +@lru_cache(1) +def get_window_topk_idxs(window_size: int, bsz: int, seqlen: int, start_pos: int): + """Which sliding-window cache slots each query attends to; -1 marks a slot holding nothing. + + The cache is a ring of `window_size` slots. Prefill needs one row per query, each seeing its own + causal window. A decode step has a single query that sees the whole ring, listed oldest first. + Order within a row does not matter to `sparse_attn`, which handles every slot independently. + """ + if start_pos == 0: + end = torch.arange(seqlen).unsqueeze(1) + idxs = (end - window_size + 1).clamp(0) + torch.arange(min(seqlen, window_size)) + idxs = torch.where(idxs > end, -1, idxs) # before the sequence started + else: + oldest = start_pos % window_size + 1 + idxs = torch.cat([torch.arange(oldest, window_size), torch.arange(oldest)]) + idxs = torch.where(idxs > start_pos, -1, idxs) # ring still filling + # sparse_attn needs real [b, m, topk] int32 memory, hence the materializing expand + return idxs.int().unsqueeze(0).expand(bsz, -1, -1).contiguous() + + +class Compressor(nn.Module): + """Pools `compress_ratio` consecutive tokens into one KV latent with a learned softmax gate. + + Returns the latent before RoPE, or None while a group is still filling up -- so during decode it + only yields every `compress_ratio` steps, holding the partial group in `kv_state`/`score_state`. + Pre-RoPE is deliberate: the indexer needs the unrotated form, so Attention rotates afterwards. + """ + + def __init__(self, args: ModelArgs, layer_id: int): + super().__init__() + compress_ratio = args.compress_ratios[layer_id] + head_dim = args.head_dim + self.compress_ratio = compress_ratio + self.head_dim = head_dim + self.norm = RMSNorm(head_dim, args.norm_eps) + # ratio 1 is a plain projection, so it stays in the checkpoint's bf16; the softmax pooling + # above ratio 1 runs in fp32, so those weights are promoted to fp32 to match + self.wkv = Linear(args.dim, head_dim, dtype=torch.float32 if compress_ratio > 1 else torch.bfloat16) + if compress_ratio > 1: + self.wgate = Linear(args.dim, head_dim, dtype=torch.float32) + # tail of an incomplete group, carried across decode steps + self.kv_state: torch.Tensor + self.score_state: torch.Tensor + state_shape = (args.max_batch_size, compress_ratio, head_dim) + self.register_buffer("kv_state", torch.zeros(state_shape, dtype=torch.float32), persistent=False) + self.register_buffer( + "score_state", torch.full(state_shape, -torch.inf, dtype=torch.float32), persistent=False + ) + + def forward(self, x: torch.Tensor, start_pos: int) -> torch.Tensor | None: + bsz, seqlen, _ = x.size() + ratio, dtype = self.compress_ratio, x.dtype + if ratio == 1: # one token per group: nothing to pool, so no gate and no fp32 + return self.norm(self.wkv(x)) + + x = x.float() + kv, score = self.wkv(x), self.wgate(x) + if start_pos == 0: + should_compress = seqlen >= ratio + remainder = seqlen % ratio + cutoff = seqlen - remainder + if remainder: # trailing partial group waits in the state + kv, self.kv_state[:bsz, :remainder] = kv.split([cutoff, remainder], dim=1) + score, self.score_state[:bsz, :remainder] = score.split([cutoff, remainder], dim=1) + kv = kv.unflatten(1, (-1, ratio)) + score = score.unflatten(1, (-1, ratio)) + kv = (kv * score.softmax(dim=2)).sum(dim=2) + else: # one token per step: fill a slot, and pool only when the group just completed + should_compress = (start_pos + 1) % ratio == 0 + slot = start_pos % ratio + self.kv_state[:bsz, slot] = kv.squeeze(1) + self.score_state[:bsz, slot] = score.squeeze(1) + if should_compress: + kv = (self.kv_state[:bsz] * self.score_state[:bsz].softmax(dim=1)).sum(dim=1, keepdim=True) + if not should_compress: + return None + return self.norm(kv.to(dtype)) + + +class Indexer(torch.nn.Module): + """Keeps the `index_topk` best compressed positions per query. + + A small side attention: fp4 query heads against one shared key per compressed position, scores + rectified then combined by `weights_proj`. With a candidate source this is the second of two + levels; `select_candidate_blocks` is the first. + """ + + def __init__(self, args: ModelArgs, layer_id: int): + super().__init__() + # the index keys are derived from the compressor's latent, so only a layer that compresses + # its own KV can produce them; every other indexer reads them from that layer's cache + self.owns_k = layer_id in args.kv_source_layers + self.compress_ratio = args.compress_ratios[layer_id] + self.is_candidate_source = layer_id == args.candidate_source_layer + self.uses_candidates = 0 <= args.candidate_source_layer < layer_id + self.candidate_topk_blocks = args.candidate_topk_blocks + self.candidate_block_size = args.candidate_block_size + self.dim = args.dim + self.n_heads = args.index_n_heads + self.n_local_heads = args.index_n_heads // world_size + self.index_head_dim = args.index_head_dim + self.rope_head_dim = args.rope_head_dim + self.index_topk = args.index_topk + self.q_lora_rank = args.q_lora_rank + self.softmax_scale = self.index_head_dim**-0.5 + self.wq_b = ColumnParallelLinear(self.q_lora_rank, self.n_heads * self.index_head_dim) + self.weights_proj = ColumnParallelLinear(self.dim, self.n_heads, dtype=torch.bfloat16) + self.freqs_cis: torch.Tensor | None = None + if self.owns_k: + self.wk = Linear(args.head_dim, self.index_head_dim, dtype=torch.bfloat16) + self.k_norm = RMSNorm(self.index_head_dim, args.norm_eps) + self.k_cache: torch.Tensor + self.register_buffer( + "k_cache", + torch.zeros(args.max_batch_size, args.max_seq_len // self.compress_ratio, args.index_head_dim), + persistent=False, + ) + + def forward(self, x: torch.Tensor, qr: torch.Tensor, latent: torch.Tensor, start_pos: int, offset: int): + """`latent` is this layer's RoPE-free compressed latent, None when this layer does not + compress or when its current group is still incomplete. An index-key owner turns it into + index keys here, which has to happen before Attention overwrites that same storage with + the RoPE'd, quantized values.""" + assert self.freqs_cis is not None + bsz, seqlen, _ = x.size() + ratio, rd, end_pos = self.compress_ratio, self.rope_head_dim, start_pos + seqlen + + # Append index keys when a compression group completes. + if self.owns_k and latent is not None: + # a latent stands for the first token of its group, so group j takes position j * ratio + freqs = ( + self.freqs_cis[: seqlen - seqlen % ratio : ratio] + if start_pos == 0 + else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0) + ) + k = self.k_norm(self.wk(latent)) + apply_rotary_emb(k[..., -rd:], freqs) + fp4_act_quant(k, fp4_block_size, True) + self.k_cache[:bsz, start_pos // ratio : start_pos // ratio + k.size(1)] = k + if self.owns_k: + shared_attn.index_k = self.k_cache + + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.index_head_dim)) + apply_rotary_emb(q[..., -rd:], self.freqs_cis[start_pos:end_pos]) + fp4_act_quant(q, fp4_block_size, True) + + index_k = shared_attn.index_k[:bsz, : end_pos // ratio] + weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads**-0.5) + index_score = torch.einsum("bshd,btd->bsht", q, index_k) + index_score = (index_score.relu_() * weights.unsqueeze(-1)).sum(dim=2) + if world_size > 1: + dist.all_reduce(index_score) + + # how many compressed positions each query can see: a block becomes visible once the query + # has passed its last token. One query per decode step, so there it is just a number. + if start_pos == 0: + compress_lens = (torch.arange(1, seqlen + 1, device=x.device) // ratio).unsqueeze(-1) + index_score.masked_fill_(torch.arange(seqlen // ratio, device=x.device) >= compress_lens, -torch.inf) + else: + compress_lens = end_pos // ratio + + if self.is_candidate_source: + shared_attn.candidates = select_candidate_blocks( + index_score, compress_lens, self.candidate_topk_blocks, self.candidate_block_size + ) + elif self.uses_candidates: + # level two: score with our own weights, but only inside the source's candidate blocks + index_score = index_score.masked_fill(~shared_attn.candidates, -torch.inf) + + # top-k by score, re-sorted into position order; unreachable -> -1, rest shifted by offset + topk = min(self.index_topk, end_pos // ratio) + idxs = index_score.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values + return torch.where(idxs < compress_lens, idxs + offset, -1).int() + + +def select_candidate_blocks( + logits: torch.Tensor, + compress_lens: torch.Tensor | int, + topk_blocks: int, + block_size: int, +) -> torch.Tensor: + """Level one of the two-level top-k: keep the `topk_blocks` highest-scoring blocks per query. + + `logits` is [..., n_positions] with positions the query cannot reach already at -inf, which is + what makes a block score of -inf mean "not reachable yet". `compress_lens` is a plain int during + decode, or broadcasts against logits' leading dims during prefill. Returns a bool mask shaped + like `logits`, so the layers consuming it just mask and never think about blocks again. + """ + width = logits.size(-1) + # score each block by its best position; -inf pads the last one out to block_size + scores = F.pad(logits, (0, -width % block_size), value=-torch.inf) + scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1) + num_blocks = scores.size(-1) + + # the block with this query's newest position is only partly filled, so pin it in: it holds the + # most recent tokens but could otherwise be outscored by an older, full block + last = (compress_lens - 1) // block_size + scores = scores.masked_fill(torch.arange(num_blocks, device=logits.device) == last, torch.inf) + + top = scores.topk(min(topk_blocks, num_blocks), dim=-1) + # fewer reachable blocks than topk_blocks means leftover picks came back -inf: drop them + keep = torch.zeros_like(scores, dtype=torch.bool).scatter_(-1, top.indices, top.values > -torch.inf) + return keep.repeat_interleave(block_size, dim=-1)[..., :width] + + +class Attention(nn.Module): + """Latent attention over two KV sources at once, concatenated into one `sparse_attn` call: a + sliding window of raw KV, plus -- when compress_ratio > 0 -- `index_topk` compressed positions + reaching further back. Q and the output projection are both low-rank, the latter grouped. + + compress_ratio > 0 does not mean the layer compresses its own KV: only kv_source_layers do, + the rest read that same cache. + """ + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__() + self.layer_id = layer_id + self.dim = args.dim + self.n_heads = args.n_heads + self.n_local_heads = args.n_heads // world_size + self.q_lora_rank = args.q_lora_rank + self.o_lora_rank = args.o_lora_rank + self.head_dim = args.head_dim + self.rope_head_dim = args.rope_head_dim + self.nope_head_dim = args.head_dim - args.rope_head_dim + self.n_groups = args.o_groups + self.n_local_groups = self.n_groups // world_size + self.window_size = args.window_size + self.compress_ratio = args.compress_ratios[layer_id] + self.eps = args.norm_eps + + self.attn_sink = nn.Parameter(torch.empty(self.n_local_heads, dtype=torch.float32)) + self.wq_a = Linear(self.dim, self.q_lora_rank) + self.q_norm = RMSNorm(self.q_lora_rank, self.eps) + self.wq_b = ColumnParallelLinear(self.q_lora_rank, self.n_heads * self.head_dim) + self.wkv = Linear(self.dim, self.head_dim) + self.kv_norm = RMSNorm(self.head_dim, self.eps) + self.wo_a = ColumnParallelLinear( + self.n_heads * self.head_dim // self.n_groups, + self.n_groups * args.o_lora_rank, + dtype=torch.bfloat16, + ) + self.wo_b = RowParallelLinear(self.n_groups * args.o_lora_rank, self.dim) + self.softmax_scale = self.head_dim**-0.5 + + is_backbone = layer_id < args.n_layers + self.is_kv_source = is_backbone and layer_id in args.kv_source_layers + self.is_index_source = is_backbone and layer_id in args.index_source_layers + self.compressor: Compressor | None = None + self.indexer: Indexer | None = None + if self.is_kv_source: + self.compressor = Compressor(args, layer_id) + if self.is_index_source: + self.indexer = Indexer(args, layer_id) + + self.window_kv_cache: torch.Tensor + self.register_buffer( + "window_kv_cache", + torch.zeros(args.max_batch_size, args.window_size, self.head_dim), + persistent=False, + ) + if self.is_kv_source: + self.compress_kv_cache: torch.Tensor + self.register_buffer( + "compress_kv_cache", + torch.zeros( + args.max_batch_size, + args.max_seq_len // self.compress_ratio, + self.head_dim, + ), + persistent=False, + ) + if self.compress_ratio: + original_seq_len, rope_theta = ( + args.original_seq_len, + args.compress_rope_theta, + ) + else: + # disable YaRN and use base rope_theta in pure sliding-window attention + original_seq_len, rope_theta = 0, args.rope_theta + freqs_cis = precompute_freqs_cis( + self.rope_head_dim, + args.max_seq_len, + original_seq_len, + rope_theta, + args.rope_factor, + args.beta_fast, + args.beta_slow, + ) + self.freqs_cis: torch.Tensor + self.register_buffer("freqs_cis", freqs_cis, persistent=False) + + def _window_kv(self, x, freqs_cis, start_pos): + """This layer's sliding-window K and the window positions every query may attend to. The K + stays fp8, quantized over the whole post-RoPE vector, RoPE tail included.""" + bsz, seqlen, _ = x.size() + win = self.window_size + kv = self.kv_norm(self.wkv(x)) + apply_rotary_emb(kv[..., -self.rope_head_dim :], freqs_cis) + act_quant(kv, fp8_block_size, scale_fmt, scale_dtype, True) + if start_pos == 0: # prefill: attend over this chunk, seeding the ring buffer for decode + if seqlen <= win: + self.window_kv_cache[:bsz, :seqlen] = kv + else: + cutoff = seqlen % win + self.window_kv_cache[:bsz, cutoff:win], self.window_kv_cache[:bsz, :cutoff] = kv[:, -win:].split( + [win - cutoff, cutoff], dim=1 + ) + window_kv = kv + else: # decode: one token into the ring buffer, attend over the whole window + self.window_kv_cache[:bsz, start_pos % win] = kv.squeeze(1) + window_kv = self.window_kv_cache[:bsz] + return window_kv, get_window_topk_idxs(win, bsz, seqlen, start_pos) + + def _compress_topk_idxs(self, x, qr, latent, start_pos, offset, compress_len): + """Which compressed positions each query attends to. Index sources run their own indexer; + the layers in between reuse the result their source published.""" + if not self.is_index_source: + return shared_attn.topk_idxs + + bsz, seqlen, _ = x.size() + if compress_len == 0: + idxs = torch.empty(bsz, seqlen, 0, dtype=torch.int32, device=x.device) + else: + assert self.indexer is not None + if self.indexer.freqs_cis is None: + self.indexer.freqs_cis = self.freqs_cis + idxs = self.indexer(x, qr, latent, start_pos, offset) + shared_attn.topk_idxs = idxs + return idxs + + def _compress_kv(self, x, qr, start_pos, offset): + """The shared compressed KV and the compressed positions every query may attend to. This + layer compresses its own KV only when it is a source; otherwise it just reads the cache.""" + bsz, seqlen, _ = x.size() + ratio = self.compress_ratio + compress_len = (start_pos + seqlen) // ratio + latent = None + if self.is_kv_source: + latent = self.compressor(x, start_pos) + shared_attn.compress_kv = self.compress_kv_cache + # the indexer needs the latent before RoPE, so it runs before the cache is written + idxs = self._compress_topk_idxs(x, qr, latent, start_pos, offset, compress_len) + if latent is not None: + # a latent stands for the first token of its group, so group j takes position j * ratio + freqs = ( + self.freqs_cis[: seqlen - seqlen % ratio : ratio] + if start_pos == 0 + else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0) + ) + apply_rotary_emb(latent[..., -self.rope_head_dim :], freqs) + # Compressed KV uses groups of 16 with E4M3 scales; the indexer uses 32 with E8M0. + fp4_act_quant(latent, 16, True, scale_dtype=torch.float8_e4m3fn) + self.compress_kv_cache[:bsz, start_pos // ratio : start_pos // ratio + latent.size(1)] = latent + # read after the write, so this does not depend on the slice aliasing the cache + return shared_attn.compress_kv[:bsz, :compress_len], idxs + + def forward(self, x: torch.Tensor, start_pos: int): + bsz, seqlen, _ = x.size() + freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen] + rd = self.rope_head_dim + + qr = self.q_norm(self.wq_a(x)) + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.head_dim)) + apply_rotary_emb(q[..., -rd:], freqs_cis) + + kv, topk_idxs = self._window_kv(x, freqs_cis, start_pos) + if self.compress_ratio: + compress_kv, compress_idxs = self._compress_kv(x, qr, start_pos, kv.size(1)) + kv = torch.cat([kv, compress_kv], dim=1) + topk_idxs = torch.cat([topk_idxs, compress_idxs], dim=-1) + + o = sparse_attn(q, kv, self.attn_sink, topk_idxs, self.softmax_scale) + apply_rotary_emb(o[..., -rd:], freqs_cis, True) + + # wo_a is block-diagonal over groups (each projects only its own heads), hence einsum not + # Linear. convert.py dequantizes it to bf16; quantize/dequantize o before einsum. + o = o.view(bsz, seqlen, self.n_local_groups, -1) + act_quant(o, fp8_block_size, scale_fmt, scale_dtype, True) + wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) + o = torch.einsum("bsgd,grd->bsgr", o, wo_a) + x = self.wo_b(o.flatten(2)) + return x + + +class Gate(nn.Module): + """MoE gating. The correction bias steers expert selection only; the routing weights come from the + unbiased scores. Image-span tokens use a separate bias (training `noaux_tc_for_vl`).""" + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__() + n_routed_experts, n_activated_experts = args.get_moe_config(layer_id) + self.dim = args.dim + self.topk = n_activated_experts + self.score_func = args.score_func + self.gate_temp = args.gate_temp + self.norm_topk_prob = args.norm_topk_prob + self.route_scale = args.route_scale + self.weight = nn.Parameter(torch.empty(n_routed_experts, args.dim)) + self.bias = nn.Parameter(torch.empty(n_routed_experts, dtype=torch.float32)) + self.bias_vl = nn.Parameter(torch.empty(n_routed_experts, dtype=torch.float32)) if args.vision_enabled else None + + def forward(self, x: torch.Tensor, image_mask: torch.Tensor | None = None) -> tuple[torch.Tensor, torch.Tensor]: + """x: [n, dim]; image_mask: [n] bool, True for tokens inside an image span.""" + scores = linear(x.float(), self.weight.float()) / self.gate_temp + if self.score_func == "softmax": + scores = scores.softmax(dim=-1) + elif self.score_func == "sigmoid": + scores = scores.sigmoid() + else: + scores = F.softplus(scores).sqrt() + bias = self.bias + if image_mask is not None and self.bias_vl is not None: + bias = torch.where(image_mask.unsqueeze(-1), self.bias_vl, bias) + # the bias picks experts but does not scale them: weights come from the raw scores + indices = (scores + bias).topk(self.topk, dim=-1)[1] + weights = scores.gather(1, indices) + if self.norm_topk_prob and self.topk > 1: + weights /= weights.sum(dim=-1, keepdim=True) + 1e-20 # not norm_eps, matches training + weights *= self.route_scale + return weights, indices + + +class Expert(nn.Module): + """One SwiGLU FFN. The clamps come straight from training, where they keep fp8/fp4 activations in + range: the up branch is clamped on both sides, the gate branch only from above.""" + + def __init__(self, dim: int, inter_dim: int, dtype=None, swiglu_limit=0.0): + super().__init__() + self.w1 = Linear(dim, inter_dim, dtype=dtype) + self.w2 = Linear(inter_dim, dim, dtype=dtype) + self.w3 = Linear(dim, inter_dim, dtype=dtype) + self.swiglu_limit = swiglu_limit + + def forward(self, x: torch.Tensor, weights: torch.Tensor | None = None) -> torch.Tensor: + dtype = x.dtype + gate = self.w1(x).float() + up = self.w3(x).float() + if self.swiglu_limit > 0: + up = torch.clamp(up, min=-self.swiglu_limit, max=self.swiglu_limit) + gate = torch.clamp(gate, max=self.swiglu_limit) + x = F.silu(gate) * up + if weights is not None: + x = weights * x + return self.w2(x.to(dtype)) + + +class MoE(nn.Module): + """Top-k routed experts plus one shared expert every token goes through. Experts are split + across ranks, so `self.experts` is None for those another rank owns.""" + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__() + n_routed_experts, n_activated_experts = args.get_moe_config(layer_id) + self.layer_id = layer_id + self.dim = args.dim + assert n_routed_experts % world_size == 0, ( + f"Number of experts must be divisible by world size (world_size={world_size})" + ) + self.n_routed_experts = n_routed_experts + self.n_local_experts = n_routed_experts // world_size + self.n_activated_experts = n_activated_experts + self.experts_start_idx = rank * self.n_local_experts + self.experts_end_idx = self.experts_start_idx + self.n_local_experts + self.gate = Gate(layer_id, args) + expert_dtype = torch.float4_e2m1fn_x2 if args.expert_dtype == "fp4" else None + self.experts = nn.ModuleList( + [ + Expert( + args.dim, + args.moe_inter_dim, + dtype=expert_dtype, + swiglu_limit=args.swiglu_limit, + ) + if self.experts_start_idx <= i < self.experts_end_idx + else None + for i in range(self.n_routed_experts) + ] + ) + assert args.n_shared_experts == 1 + self.shared_experts = Expert(args.dim, args.moe_inter_dim, swiglu_limit=args.swiglu_limit) + + def forward(self, x: torch.Tensor, image_mask: torch.Tensor | None = None) -> torch.Tensor: + shape = x.size() + x = x.view(-1, self.dim) + weights, indices = self.gate(x, None if image_mask is None else image_mask.flatten()) + y = torch.zeros_like(x, dtype=torch.float32) + counts = torch.bincount(indices.flatten(), minlength=self.n_routed_experts).tolist() + for i in range(self.experts_start_idx, self.experts_end_idx): + if counts[i] == 0: + continue + expert = self.experts[i] + idx, top = torch.where(indices == i) + y[idx] += expert(x[idx], weights[idx, top, None]) + if world_size > 1: + dist.all_reduce(y) + y += self.shared_experts(x) + return y.type_as(x).view(shape) + + +class Block(nn.Module): + """A block whose residual stream is `hc_mult` parallel copies (Hyper-Connections). + + Attention and FFN each sit between `hc_pre` (collapse the copies into one sublayer input) and + `hc_post` (expand back out, mixing the residual in through `comb`). `hc_mixes` derives all three + coefficient sets from the stream itself, `comb` made doubly stochastic by Sinkhorn. + + The coefficients a sublayer computes are used by the *next* one -- see `forward`. + """ + + attention_cls = Attention + + def __init__( + self, + layer_id: int, + args: ModelArgs, + engram_layout: EngramLayout | None = None, + ): + super().__init__() + self.layer_id = layer_id + self.norm_eps = args.norm_eps + self.attn = self.attention_cls(layer_id, args) + self.ffn = MoE(layer_id, args) + self.engram = None + if engram_layout is not None and layer_id in engram_layout.layer_ids: + self.engram = Engram(args, layer_id, engram_layout) + self.attn_norm = RMSNorm(args.dim, self.norm_eps) + self.ffn_norm = RMSNorm(args.dim, self.norm_eps) + self.hc_mult = hc_mult = args.hc_mult + self.hc_sinkhorn_iters = args.hc_sinkhorn_iters + self.hc_eps = args.hc_eps + mix_hc = (2 + hc_mult) * hc_mult + hc_dim = hc_mult * args.dim + with set_dtype(torch.float32): + self.hc_attn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim)) + self.hc_ffn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim)) + self.hc_attn_base = nn.Parameter(torch.empty(mix_hc)) + self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc)) + self.hc_attn_scale = nn.Parameter(torch.empty(3)) + self.hc_ffn_scale = nn.Parameter(torch.empty(3)) + + def hc_mixes(self, x: torch.Tensor, hc_fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor): + """x: [b,s,hc,d], hc_fn: [mix_hc, hc*d], hc_scale: [3], hc_base: [mix_hc]. Returns the + pre / post / comb coefficients, split out of one projection of the flattened stream.""" + # normalized over the whole flattened hc*d stream, one statistic per token + x = x.flatten(2).float() + rsqrt = torch.rsqrt(x.square().mean(-1, keepdim=True) + self.norm_eps) + mixes = F.linear(x, hc_fn) * rsqrt + return hc_split_sinkhorn(mixes, hc_scale, hc_base, self.hc_mult, self.hc_sinkhorn_iters, self.hc_eps) + + def hc_pre(self, x: torch.Tensor, pre_mix: torch.Tensor): + """Collapse the hc copies into one, weighted by pre_mix. [b,s,hc,d] x [b,s,hc] -> [b,s,d]""" + y = torch.sum(pre_mix.unsqueeze(-1) * x.float(), dim=2) + return y.to(x.dtype) + + def hc_post(self, x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor): + """Expand the sublayer output back to hc copies and mix the residual in through `comb`. + x: [b,s,d], residual: [b,s,hc,d], post: [b,s,hc], comb: [b,s,hc,hc] -> [b,s,hc,d]""" + y = post.unsqueeze(-1) * x.unsqueeze(-2) + torch.sum(comb.unsqueeze(-1) * residual.unsqueeze(-2), dim=2) + return y.type_as(x) + + def forward( + self, + x: torch.Tensor, + start_pos: int, + pre_mix: torch.Tensor, + image_mask: torch.Tensor | None, + *attn_args, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`pre_mix` collapses the hc_mult copies down to one input for this block's attention. Each + sub-block's own `hc_mixes` produces the mix for the *next* one, so attention uses what the + previous layer's FFN produced and the FFN uses what this attention produced. + + image_mask: [b, s] bool, True inside image spans (selects the VL routing bias).""" + residual = x + attn_pre, attn_post, attn_comb = self.hc_mixes(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base) + x = self.hc_pre(x, pre_mix) + x = self.attn_norm(x) + x = self.attn(x, start_pos, *attn_args) + x = self.hc_post(x, residual, attn_post, attn_comb) + + residual = x + ffn_pre, ffn_post, ffn_comb = self.hc_mixes(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base) + x = self.hc_pre(x, attn_pre) + x = self.ffn_norm(x) + x = self.ffn(x, image_mask) + x = self.hc_post(x, residual, ffn_post, ffn_comb) + return x, ffn_pre + + +class ParallelHead(nn.Module): + def __init__(self, vocab_size: int, dim: int, norm_eps: float = 1e-6, hc_eps: float = 1e-6): + super().__init__() + self.vocab_size = vocab_size + self.dim = dim + self.norm_eps = norm_eps + self.hc_eps = hc_eps + self.part_vocab_size = vocab_size // world_size + # bf16 in the checkpoint, kept as fp32 here so the logits come out in fp32 directly + self.weight = nn.Parameter(torch.empty(self.part_vocab_size, self.dim, dtype=torch.float32)) + + def forward(self, x: torch.Tensor, full_logits=False): + """x: [b, s, d]. Generation only needs the last position, so that is the default.""" + if not full_logits: + x = x[:, -1] + logits = F.linear(x.float(), self.weight) + if world_size > 1: + all_logits = [torch.empty_like(logits) for _ in range(world_size)] + dist.all_gather(all_logits, logits) + logits = torch.cat(all_logits, dim=-1) + return logits + + +@lru_cache(1) +def get_dspark_topk_idxs(window_size: int, bsz: int, block_size: int, start_pos: int): + assert start_pos > 0 + matrix = torch.cat( + [ + torch.arange(min(window_size, start_pos + 1)), + window_size + torch.arange(block_size), + ] + ) + return matrix.int().view(1, 1, -1).expand(bsz, block_size, -1).contiguous() + + +class DSparkAttention(Attention): + def forward(self, x: torch.Tensor, start_pos: int, main_x: torch.Tensor): + assert self.compress_ratio == 0 + bsz, seqlen, _ = main_x.size() + win = self.window_size + rd = self.rope_head_dim + + main_freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen] + main_kv = self.kv_norm(self.wkv(main_x)) + apply_rotary_emb(main_kv[..., -rd:], main_freqs_cis) + act_quant(main_kv, fp8_block_size, scale_fmt, scale_dtype, True) + + if start_pos == 0: + if seqlen <= win: + self.window_kv_cache[:bsz, :seqlen] = main_kv + else: + cutoff = seqlen % win + self.window_kv_cache[:bsz, cutoff:win], self.window_kv_cache[:bsz, :cutoff] = main_kv[:, -win:].split( + [win - cutoff, cutoff], dim=1 + ) + return x + + bsz, block_size, _ = x.size() + freqs_cis = self.freqs_cis[start_pos + seqlen : start_pos + seqlen + block_size] + + qr = self.q_norm(self.wq_a(x)) + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.head_dim)) + apply_rotary_emb(q[..., -rd:], freqs_cis) + kv = self.kv_norm(self.wkv(x)) + apply_rotary_emb(kv[..., -rd:], freqs_cis) + act_quant(kv, fp8_block_size, scale_fmt, scale_dtype, True) + + topk_idxs = get_dspark_topk_idxs(win, bsz, block_size, start_pos) + self.window_kv_cache[:bsz, start_pos % win] = main_kv.squeeze(1) + kv = torch.cat([self.window_kv_cache[:bsz], kv], dim=1) + o = sparse_attn(q, kv, self.attn_sink, topk_idxs, self.softmax_scale) + apply_rotary_emb(o[..., -rd:], freqs_cis, True) + + o = o.view(bsz, block_size, self.n_local_groups, -1) + act_quant(o, fp8_block_size, scale_fmt, scale_dtype, True) + wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) + o = torch.einsum("bsgd,grd->bsgr", o, wo_a) + x = self.wo_b(o.flatten(2)) + return x + + +class DSparkMarkovHead(nn.Module): + def __init__(self, vocab_size: int, dspark_markov_rank: int): + super().__init__() + self.embed = ParallelEmbedding(vocab_size, dspark_markov_rank) + self.head = ParallelHead(vocab_size, dspark_markov_rank) + + def forward(self, token_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + embed = self.embed(token_ids) + logits = self.head(embed, full_logits=True) + return logits, embed + + +class DSparkConfidenceHead(nn.Module): + def __init__(self, input_dim: int): + super().__init__() + # proj in the checkpoint is stored in bf16, while the parameter here is stored in fp32 for fp32 confidence score. + self.proj = Linear(input_dim, 1, dtype=torch.float32) + + def forward(self, hidden: torch.Tensor, markov_embed: torch.Tensor): + hidden = torch.cat([hidden, markov_embed], dim=-1) + return self.proj(hidden.float()).squeeze(-1) + + +class DSparkBlock(Block): + """DSpark stage stored under the mtp.* checkpoint namespace.""" + + attention_cls = DSparkAttention + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__(layer_id, args) + stage_id = layer_id - args.n_layers + self.block_size = args.dspark_block_size + self.noise_token_id = args.dspark_noise_token_id + self.temperature = args.temperature + if stage_id == 0: + assert len(args.dspark_target_layer_ids) > 0, "DSpark needs target layers" + self.main_proj = Linear(args.dim * len(args.dspark_target_layer_ids), args.dim) + self.main_norm = RMSNorm(args.dim, args.norm_eps) + if stage_id == args.n_mtp_layers - 1: + self.norm = RMSNorm(args.dim, args.norm_eps) + self.markov_head = DSparkMarkovHead(args.vocab_size, args.dspark_markov_rank) + self.confidence_head = DSparkConfidenceHead(args.dim + args.dspark_markov_rank) + self.embed: ParallelEmbedding | None = None + self.head: ParallelHead | None = None + + def forward(self, x: torch.Tensor, start_pos: int, pre_mix: torch.Tensor, main_x: torch.Tensor): + if start_pos == 0: + self.attn(x, start_pos, main_x) # prefill only seeds the window KV cache + return x, pre_mix + return super().forward(x, start_pos, pre_mix, None, main_x) # drafts are text: no VL bias + + def forward_embed(self, main_hidden: torch.Tensor, input_ids: torch.Tensor): + assert self.embed is not None + main_x = self.main_norm(self.main_proj(main_hidden)) + draft_input_ids = input_ids.new_full([input_ids.size(0), self.block_size], self.noise_token_id) + draft_input_ids[:, 0] = input_ids + x = self.embed(draft_input_ids) + x = x.unsqueeze(2).repeat(1, 1, self.hc_mult, 1) + return x, main_x + + def forward_head( + self, + x: torch.Tensor, + pre_mix: torch.Tensor, + input_ids: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert self.head is not None + x = self.hc_pre(x, pre_mix) + logits = self.head(self.norm(x), full_logits=True) + output_ids = input_ids.new_empty(input_ids.size(0), self.block_size + 1) + output_ids[:, 0] = input_ids + markov_embeds = [] + for i in range(self.block_size): + logits_bias, markov_embed = self.markov_head(output_ids[:, i]) + logits[:, i].add_(logits_bias) + markov_embeds.append(markov_embed) + output_ids[:, i + 1] = sample(logits[:, i], self.temperature) + markov_embed = torch.stack(markov_embeds, dim=1) + confidence = self.confidence_head(x, markov_embed) + return output_ids, logits, confidence + + +def make_identity_pre_mix(x: torch.Tensor, hc_mult: int) -> torch.Tensor: + """initial one-hot mix""" + pre_mix = x.new_zeros(x.size(0), x.size(1), hc_mult, dtype=torch.float32) + pre_mix[:, :, 0] = 1.0 + return pre_mix + + +class SharedAttentionRuntime: + """What attention layers hand down the stack instead of recomputing. Layers run in order and + every source writes before its consumers read, so one slot each is enough and nothing needs + resetting between forwards. Sources: compress_kv and index_k from kv_source_layers, topk_idxs + from index_source_layers, candidates from candidate_source_layer.""" + + def __init__(self): + self.compress_kv: torch.Tensor | None = None + self.index_k: torch.Tensor | None = None + self.topk_idxs: torch.Tensor | None = None + self.candidates: torch.Tensor | None = None + + +# Only ever one model per process, same as world_size / rank / default_dtype above. +shared_attn = SharedAttentionRuntime() + + +class Transformer(nn.Module): + """DeepSeek-V4.1: embed -> expand to hc_mult copies -> blocks -> collapse -> logits. Building + this sets the globals at the top of the file. The tokenizer only feeds the engram token map.""" + + def __init__(self, args: ModelArgs, tokenizer=None): + global world_size, rank, default_dtype + world_size = dist.get_world_size() if dist.is_initialized() else 1 + rank = dist.get_rank() if dist.is_initialized() else 0 + default_dtype = torch.float8_e4m3fn if args.dtype == "fp8" else torch.bfloat16 + super().__init__() + self.max_seq_len = args.max_seq_len + self.temperature = args.temperature + self.norm_eps = args.norm_eps + self.hc_eps = args.hc_eps + self.engram_layout = EngramLayout.from_args(args) + self.engram_hash = ( + NgramHashState(args, self.engram_layout, tokenizer) if self.engram_layout is not None else None + ) + self.embed = ParallelEmbedding(args.vocab_size, args.dim) + self.layers = torch.nn.ModuleList() + for layer_id in range(args.n_layers): + self.layers.append(Block(layer_id, args, self.engram_layout)) + self.norm = RMSNorm(args.dim, self.norm_eps) + self.head = ParallelHead(args.vocab_size, args.dim, self.norm_eps, self.hc_eps) + self.mtp = torch.nn.ModuleList() + self.target_layer_ids = args.dspark_target_layer_ids + if args.dspark_block_size: + for layer_id in range(args.n_mtp_layers): + self.mtp.append(DSparkBlock(args.n_layers + layer_id, args)) + self.mtp[-1].embed = self.embed + self.mtp[-1].head = self.head + self.hc_mult = args.hc_mult + self.vision = None + if args.vision_enabled: + self.vision = ViT(args) + self.aligner = Aligner(args) + # learned embeddings for the image span delimiters + self.image_start = nn.Parameter(torch.empty(args.dim)) + self.image_end = nn.Parameter(torch.empty(args.dim)) + self.image_newline = nn.Parameter(torch.empty(args.dim)) + + @torch.inference_mode() + def encode_image(self, patches: torch.Tensor, n_vit_h: int, n_vit_w: int) -> torch.Tensor: + return self.aligner(self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w) + + def merge_image_embeddings(self, images, h: torch.Tensor): + """Overwrite each image's token span in h with its ViT/aligner features. The IMAGE slots take + the aligner rows in row-major order; the span delimiters take learned embeddings.""" + for i, sample in enumerate(images): + for img in sample or (): + types = img.types.to(h.device) + span = h[i, img.start : img.start + types.numel()] + span[types == IMAGE_START] = self.image_start.to(h.dtype) + span[types == IMAGE_END] = self.image_end.to(h.dtype) + span[types == IMAGE_NEW_LINE] = self.image_newline.to(h.dtype) + embeds = self.encode_image(img.patches.to(h.device), img.n_vit_h, img.n_vit_w) + span[types == IMAGE] = embeds.to(h.dtype) + + @torch.inference_mode() + def forward( + self, input_ids: torch.Tensor, start_pos: int = 0, images=None, token_types: torch.Tensor | None = None + ): + """input_ids: [b, s], every entry a real token id -- generate.py only ever passes positions it + has already filled, so the padding it uses internally never reaches here. token_types / + images carry the VL inputs built by image_processor.prepare_vl_inputs; image spans must lie + inside the first (start_pos 0) chunk.""" + image_mask = None if token_types is None else token_types >= 0 # TEXT is -1 + # image tokens take no part in an n-gram and get no engram contribution; text-only needs no mask + engram_mask = None if image_mask is None else ~image_mask + engram_hashes = self.engram_hash(input_ids, start_pos, engram_mask) if self.engram_hash is not None else None + h = self.embed(input_ids) + if images is not None: + assert start_pos == 0, "image spans must be prefilled in a single chunk" + self.merge_image_embeddings(images, h) + # Expand to hc_mult copies for Hyper-Connections + h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1) + main_hiddens = [] + pre_mix = make_identity_pre_mix(h, self.hc_mult) + for i, layer in enumerate(self.layers): + if layer.engram is not None: + h = layer.engram(h, engram_hashes[:, :, layer.engram.layer_hash_index, :], engram_mask) + # the MTP head reads the attention input of its target layers, not their output + if i in self.target_layer_ids: + main_hiddens.append(h.mean(dim=2)) + h, pre_mix = layer(h, start_pos, pre_mix, image_mask) + h = layer.hc_pre(h, pre_mix) + logits = self.head(self.norm(h)) + output_ids = sample(logits, self.temperature) + main_hidden = torch.cat(main_hiddens, dim=-1) if main_hiddens else None + return output_ids, logits, main_hidden + + @torch.inference_mode() + def forward_spec(self, input_ids: torch.Tensor, main_hidden: torch.Tensor, start_pos: int = 0): + h, main_x = self.mtp[0].forward_embed(main_hidden, input_ids) + pre_mix = make_identity_pre_mix(h, self.hc_mult) + for layer in self.mtp: + h, pre_mix = layer(h, start_pos, pre_mix, main_x) + if start_pos == 0: + return None + return self.mtp[-1].forward_head(h, pre_mix, input_ids) + + +def sample(logits, temperature: float = 1.0): + """Gumbel-max trick: equivalent to multinomial sampling but faster on GPU, + since it avoids the GPU-to-CPU sync in torch.multinomial.""" + if temperature == 0: + return logits.argmax(dim=-1) + logits = logits / max(temperature, 1e-5) + probs = torch.softmax(logits, dim=-1, dtype=torch.float32) + return probs.div_(torch.empty_like(probs).exponential_(1)).argmax(dim=-1) + + +if __name__ == "__main__": + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") + torch.manual_seed(0) + args = ModelArgs(dspark_block_size=6, dspark_target_layer_ids=(3, 4)) + x = torch.randint(0, args.vocab_size, (2, 150)) + model = Transformer(args) + + output_ids, logits, main_hidden = model(x[:, :128]) + model.forward_spec(output_ids, main_hidden) + for i in range(128, 150): + output_ids, logits, main_hidden = model(x[:, i : i + 1], i) + result = model.forward_spec(output_ids, main_hidden, i) + assert result is not None + output_ids, logits, confidence = result diff --git a/tests/test_runpod_v41_stream.py b/tests/test_runpod_v41_stream.py new file mode 100644 index 0000000..742f713 --- /dev/null +++ b/tests/test_runpod_v41_stream.py @@ -0,0 +1,266 @@ +"""The layer-streamed driver against DeepSeek's own Transformer.forward, on CPU at toy size. + +The real ``model.py`` and ``engram.py`` (tests/fixtures/v41_reference) run with +plain-torch kernels. The toy model keeps every feature that makes streaming +subtle: + +- hyper-connections; +- compressed KV shared from source layers; +- an indexer shared across layers; +- a candidate-block source and a layer consuming it; +- an Engram layer; +- a MoE. +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + +REPO = Path(__file__).resolve().parents[1] +REF_DIR = REPO / "tests" / "fixtures" / "v41_reference" +KIT = REPO / "scripts" / "runpod" / "v41_reap" +sys.path.insert(0, str(KIT)) + +VOCAB = 96 + + +def _install_stubs() -> None: + import torch_kernel + + sys.modules["kernel"] = torch_kernel + image = types.ModuleType("image_processor") + image.IMAGE = image.IMAGE_END = image.IMAGE_NEW_LINE = image.IMAGE_START = None + vision = types.ModuleType("vision") + vision.Aligner = vision.ViT = None + sys.modules.setdefault("image_processor", image) + sys.modules.setdefault("vision", vision) + try: + import sympy + except ImportError: # engram only needs isprime + sympy = types.ModuleType("sympy") + sympy.isprime = lambda n: n > 1 and all(n % p for p in range(2, int(n**0.5) + 1)) + sys.modules["sympy"] = sympy + + +@pytest.fixture(scope="module") +def toy(tmp_path_factory): + _install_stubs() + import stream + + ref = stream.import_reference(REF_DIR) + import engram + + engram.build_compressed_token_map = lambda tokenizer: (list(range(VOCAB)), VOCAB) + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(0) + + base = dict( + max_batch_size=1, max_seq_len=64, dtype="bf16", expert_dtype=None, vocab_size=VOCAB, + dim=64, moe_inter_dim=32, n_layers=6, n_mtp_layers=0, n_heads=4, n_routed_experts=8, + n_activated_experts=2, route_scale=1.5, swiglu_limit=10.0, q_lora_rank=32, head_dim=32, + rope_head_dim=8, norm_eps=1e-6, o_groups=2, o_lora_rank=16, window_size=8, + compress_ratios=(0, 2, 2, 1, 1, 0), kv_source_layers=(1, 3), index_source_layers=(1, 3, 4), + index_n_heads=2, index_head_dim=16, index_topk=4, candidate_source_layer=3, + candidate_topk_blocks=2, candidate_block_size=2, original_seq_len=64, + engram_layer_ids=(2,), engram_max_ngram_size=3, engram_vocab_size=40, engram_n_heads=2, + engram_head_dim=32, engram_pad_id=2, engram_compressed_vocab_size=VOCAB, + ) + layout = ref.EngramLayout.from_args(types.SimpleNamespace(**{**base, "engram_num_embeddings": (0,)})) + rows = int(sum(p for per_ngram in layout.primes[0] for p in per_ngram)) + args = ref.ModelArgs(**base, engram_num_embeddings=(rows,)) + model = ref.Transformer(args, tokenizer=None) + + gen = torch.Generator().manual_seed(1) + with torch.no_grad(): + for name, p in model.named_parameters(): + shape = p.shape + if p.dtype == torch.float8_e4m3fn: + p.copy_((torch.randn(shape, generator=gen) * 0.5).to(p.dtype)) + elif p.dtype == torch.float8_e8m0fnu: + p.copy_(torch.full(shape, 0.5).to(p.dtype)) + elif "norm" in name or name.endswith(("q_weight", "k_weight")): + p.copy_(1 + 0.1 * torch.randn(shape, generator=gen)) + elif "hc_" in name and "scale" in name: + p.copy_(0.5 + 0.1 * torch.randn(shape, generator=gen)) + else: + fan_in = shape[-1] if len(shape) > 1 else 1 + p.copy_(torch.randn(shape, generator=gen) / max(fan_in, 1) ** 0.5) + + from safetensors.torch import save_file + + ckpt = tmp_path_factory.mktemp("hf") + sd = {k: v.contiguous() for k, v in model.state_dict().items()} + names = sorted(sd) + half = len(names) // 2 + shards = {"model-00001.safetensors": names[:half], "model-00002.safetensors": names[half:]} + weight_map = {} + for shard, keys in shards.items(): + save_file({k: sd[k] for k in keys}, str(ckpt / shard)) + weight_map.update({k: shard for k in keys}) + (ckpt / stream.INDEX).write_text(json.dumps({"weight_map": weight_map})) + + records = [torch.randint(0, VOCAB, (1, n), generator=gen) for n in (37, 23, 57)] + return types.SimpleNamespace(ref=ref, stream=stream, args=args, model=model, ckpt=ckpt, records=records) + + +def _reference_hidden(toy, tokens): + captured = {} + handle = toy.model.norm.register_forward_hook(lambda m, i, o: captured.setdefault("h", o)) + try: + with torch.inference_mode(): + toy.model.forward(tokens, 0) + finally: + handle.remove() + return captured["h"] + + +def _streamer(toy): + stream = toy.stream + ckpt = stream.HFCheckpoint(toy.ckpt) + rows = {layer: stream.EngramRows.from_checkpoint(ckpt, layer, toy.ref.fp8_block_size) for layer in toy.args.engram_layer_ids} + return stream.LayerStreamer(toy.ref, toy.args, ckpt.load, rows, toy.model.engram_hash, "cpu") + + +@pytest.mark.parametrize("grouping", ["one-moe-batch", "moe-per-record"]) +def test_streamed_forward_matches_reference(toy, grouping): + reference = [_reference_hidden(toy, t) for t in toy.records] + streamer = _streamer(toy) + items = [ + toy.stream.Item(tokens=t, group=0 if grouping == "one-moe-batch" else i) for i, t in enumerate(toy.records) + ] + with torch.inference_mode(): + streamer.forward(items) + streamed = [streamer.final_hidden(item) for item in items] + for want, got in zip(reference, streamed): + assert got.shape == want.shape + torch.testing.assert_close(got.float(), want.float(), rtol=0, atol=0) + + +def test_record_alone_equals_record_in_chunk(toy): + """The shared_attn save/restore: a record's output does not depend on its chunk-mates.""" + + streamer = _streamer(toy) + alone = [toy.stream.Item(tokens=toy.records[2], group=0)] + crowd = [toy.stream.Item(tokens=t, group=i) for i, t in enumerate(toy.records)] + with torch.inference_mode(): + streamer.forward(alone) + streamer.forward(crowd) + torch.testing.assert_close(streamer.final_hidden(crowd[2]), streamer.final_hidden(alone[0]), rtol=0, atol=0) + + +def test_pack_bits_round_trip(toy): + mask = torch.rand(3, 5, 13) > 0.5 + assert torch.equal(toy.stream.unpack_bits(toy.stream.pack_bits(mask)), mask) + + +# --------------------------------------------------------------------------- the two sweeps (collect.py) + +LAYERS = list(range(6)) +DIMS = {"hidden": 64, "down": 32} + + +def _record(tokens, name, split): + ids = tokens[0].tolist() + return {"id": f"{name}-{len(ids)}", "session": name, "split": split, "domain": "code", "token_ids": ids, "target_spans": [[3, len(ids)]]} + + +def _reference_stats(toy, collect, records, dims): + stats = collect.SessionStats(torch=torch, n_layers=6, n_experts=8, dims=dims, device="cpu") + original = toy.ref.MoE.forward + toy.ref.MoE.forward = collect.make_moe_forward(toy.ref, torch, torch.nn.functional, None, {"rank": 0, "stats": stats}) + try: + hidden = [_reference_hidden(toy, torch.as_tensor([r["token_ids"]])) for r in records] + finally: + toy.ref.MoE.forward = original + return stats.arrays(LAYERS), hidden + + +def _sweep(toy, collect, monkeypatch, chunk, split, students=None): + streamer = _streamer(toy) + monkeypatch.setattr(toy.ref.MoE, "forward", toy.ref.MoE.forward) # install_hooks patches it; undo at teardown + state = collect.install_hooks(streamer, top_k=16, rows_per_slice=7) + return collect.run_chunk(streamer, state, chunk, split=split, students=students or {}, dims=DIMS, device="cpu"), streamer + + +def _assert_stats_match(got, want): + assert set(got) == set(want) + for key, value in want.items(): + if np.issubdtype(value.dtype, np.integer): + np.testing.assert_array_equal(got[key], value, err_msg=key) + else: + np.testing.assert_allclose(got[key], value, rtol=1e-6, atol=1e-12, err_msg=key) + + +def test_calibration_sweep_matches_reference(toy, monkeypatch): + import collect + + r0, r1, r2 = toy.records + chunk = [("cal-0", [_record(r0, "cal-0", "calibration"), _record(r1, "cal-0", "calibration")]), + ("cal-1", [_record(r2, "cal-1", "calibration")])] + want = {name: _reference_stats(toy, collect, records, DIMS)[0] for name, records in chunk} + results, _ = _sweep(toy, collect, monkeypatch, chunk, "calibration") + assert set(results) == {("teacher", "cal-0"), ("teacher", "cal-1")} + for name, _ in chunk: + _assert_stats_match(results[("teacher", name)], want[name]) + assert results[("teacher", "cal-0")]["routes__expert_ids"].shape == (60, 6, 2) + + +def test_eval_sweep_teacher_and_students_match_reference(toy, monkeypatch): + import collect + + F = torch.nn.functional + chunk = [(f"eval-{i}", [_record(t, f"eval-{i}", "eval")]) for i, t in enumerate(toy.records[:2])] + gen = torch.Generator().manual_seed(7) + half = {"survivors": {str(layer): sorted(torch.randperm(8, generator=gen)[:4].tolist()) for layer in LAYERS}} + students = {"8": collect.survivor_masks({"survivors": {str(layer): list(range(8)) for layer in LAYERS}}, 8), + "4": collect.survivor_masks(half, 8)} + + want = {name: _reference_stats(toy, collect, records, {}) for name, records in chunk} + with torch.no_grad(): + full = {layer: toy.model.layers[layer].ffn.gate.bias.clone() for layer in LAYERS} + for layer, mask in students["4"].items(): + toy.model.layers[layer].ffn.gate.bias[mask] = float("-inf") + masked = {name: _reference_hidden(toy, torch.as_tensor([records[0]["token_ids"]])) for name, records in chunk} + for layer, bias in full.items(): + toy.model.layers[layer].ffn.gate.bias.copy_(bias) + + results, _ = _sweep(toy, collect, monkeypatch, chunk, "eval", students) + assert set(results) == {(s, name) for s in ("teacher", "8", "4") for name, _ in chunk} + weight = toy.model.head.weight.detach() + for name, records in chunk: + stats, (hidden,) = want[name] + teacher = results[("teacher", name)] + _assert_stats_match({k: v for k, v in teacher.items() if k in stats}, stats) + assert not any(k.startswith("importance_sum") for k in teacher) + + positions = collect.target_positions(records[0]) + targets = torch.as_tensor(records[0]["token_ids"])[positions + 1] + logits = F.linear(hidden[0, positions].float(), weight) + np.testing.assert_allclose(teacher["logsumexp"], torch.logsumexp(logits, -1).numpy(), rtol=1e-6) + np.testing.assert_allclose(teacher["target_logit"], logits.gather(1, targets[:, None])[:, 0].numpy(), rtol=1e-6) + np.testing.assert_array_equal(teacher["positions"], positions) + + everyone = results[("8", name)] + np.testing.assert_allclose(everyone["kl_on_teacher_topk"], 0.0, atol=2e-3) + assert everyone["top1_agree"].all() + + pruned = F.linear(masked[name][0, positions].float(), weight) + nll = torch.logsumexp(pruned, -1) - pruned.gather(1, targets[:, None])[:, 0] + np.testing.assert_allclose(results[("4", name)]["student_nll"], nll.numpy(), rtol=1e-5) + assert not np.allclose(results[("4", name)]["student_nll"], everyone["student_nll"]) + + +def test_chunk_sessions_keeps_sessions_whole(): + import collect + + sessions = [(n, [{"token_ids": [0] * size}]) for n, size in (("a", 5), ("b", 5), ("c", 20), ("d", 1))] + chunks = collect.chunk_sessions(sessions, budget=24, streams=2) + assert [[n for n, _ in c] for c in chunks] == [["a", "b"], ["c"], ["d"]] From b864452b4fdd2254aadb5dab94441c2d8bf3e10e Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:11:52 -0800 Subject: [PATCH 22/74] runpod: pod.sh fixes from the first H100 pod (hf includes, self-stop, smoke stage, prune 192) --- scripts/runpod/v41_reap/pod.sh | 63 +++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/scripts/runpod/v41_reap/pod.sh b/scripts/runpod/v41_reap/pod.sh index 7d0ec53..8e13984 100755 --- a/scripts/runpod/v41_reap/pod.sh +++ b/scripts/runpod/v41_reap/pod.sh @@ -7,7 +7,8 @@ # pod.sh gpu GPU pod (1-8 GPUs, 80 GB+ each): calibration sweep, reap-select, # eval sweep (teacher + every candidate side by side), scores # pod.sh prune KEEP CPU pod: write the pruned HF checkpoint for the Mac to pull -# pod.sh all one GPU pod does everything: fetch || pack, gpu, prune $PRUNE +# pod.sh smoke GPU pod: the longest calibration record through all 40 layers; memory check +# pod.sh all one GPU pod does everything: fetch || pack, smoke, gpu, prune $PRUNE # # Everything lives under /workspace (a network volume, or the pod's own volume # disk in single-pod mode). @@ -20,9 +21,11 @@ KIT=$W/keep/scripts/runpod/v41_reap HF=$W/v41-hf OUT=$W/out CANDIDATES=${CANDIDATES:-"256 192 128 96"} -PRUNE=${PRUNE:-"192 96"} # REAP 50% and REAP 75% of 384 experts -MAX_GPU_HOURS=${MAX_GPU_HOURS:-8} # hard stop for the GPU pod (1xH100 ~ $2.69/h, 1xH200 $3.59-5.29/h) -export PYTHONPATH=$W/keep/src HF_HUB_ENABLE_HF_TRANSFER=1 +PRUNE=${PRUNE:-"192"} # REAP 50%. REAP 75% (96) after the Mac pulls 192: 1.2 TB holds one +MAX_GPU_HOURS=${MAX_GPU_HOURS:-8} # hard stop for the GPU pod (1xH100 SXM secure $3.99/h) +export PYTHONPATH=$W/keep/src +# ssh sessions miss the pod's env (RUNPOD_POD_ID, the pod-scoped RUNPOD_API_KEY): load it, never print it +if [[ -z "${RUNPOD_POD_ID:-}" && -r /etc/rp_environment ]]; then source /etc/rp_environment; fi watchdog() { # whatever happens next, the pod stops at MAX_GPU_HOURS; a failed stage stops it at once [[ -n "${WATCHDOG_STARTED:-}" ]] && return @@ -36,15 +39,17 @@ log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$W/pod.log"; } stop_pod() { log "stopping pod: $*" if [[ -n "${RUNPOD_POD_ID:-}" ]] && command -v runpodctl >/dev/null; then - runpodctl stop pod "$RUNPOD_POD_ID" || log "runpodctl stop FAILED: stop the pod by hand" + runpodctl pod stop "$RUNPOD_POD_ID" || runpodctl stop pod "$RUNPOD_POD_ID" || log "runpodctl stop FAILED: stop the pod by hand" else log "no runpodctl/RUNPOD_POD_ID: stop the pod by hand" fi } fetch_small() { # tokenizer, configs, encoding/ and inference/: all pack and collect need besides weights - pip install -q "huggingface_hub[hf_transfer]>=0.34" "datasets>=3" tokenizers - hf download "$MODEL_ID" --revision "$REVISION" --local-dir "$HF" --include "*.json" "encoding/*" "inference/*" + pip install -q "huggingface_hub>=0.34" "datasets>=3" tokenizers + hf download "$MODEL_ID" --revision "$REVISION" --local-dir "$HF" \ + --include "*.json" --include "encoding/*" --include "inference/*" + test -f "$HF/model.safetensors.index.json" || { log "fetch_small: no index json in $HF"; return 1; } } fetch() { @@ -95,14 +100,47 @@ sweep() { # one collect.py per GPU, sessions sharded across them; extra args go done } +gpu_setup() { + [[ -f "$W/pip-freeze-gpu.txt" ]] && return + pip install -q -r "$HF/inference/requirements.txt" + pip freeze > "$W/pip-freeze-gpu.txt" + nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$W/pod.log" +} + +smoke() { # the longest calibration record through all 40 layers, before any sweep + watchdog + gpu_setup + python3 - "$W/pack/v41-pack.jsonl" "$W/smoke/pack.jsonl" <<'PY' +import json, pathlib, sys +rows = [json.loads(line) for line in open(sys.argv[1]) if line.strip()] +row = max((r for r in rows if r["split"] == "calibration"), key=lambda r: len(r["token_ids"])) +row["session"] = "smoke" +pathlib.Path(sys.argv[2]).parent.mkdir(parents=True, exist_ok=True) +pathlib.Path(sys.argv[2]).write_text(json.dumps(row) + "\n") +print("smoke record", row["id"], len(row["token_ids"]), "tokens") +PY + rm -rf "$W/smoke/calibration" "$W/smoke"/progress-* + python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/smoke/pack.jsonl" --out "$W/smoke" --sweep calibration \ + 2>&1 | tee "$W/smoke.log" + python3 - "$W/smoke" <<'PY' +import glob, json, sys +import numpy as np +(entry,) = [json.loads(line) for f in glob.glob(sys.argv[1] + "/progress-*.jsonl") for line in open(f)] +with np.load(sys.argv[1] + "/calibration/sessions/smoke.npz") as z: + slots = z["reap__routed_slots"] + assert (slots == slots[0]).all() and slots[0] > 0, slots + assert np.isfinite(z["reap__gate_norm_sum"]).all() and np.isfinite(z["importance_sum__hidden"]).all() +print("smoke ok:", json.dumps(entry)) +sys.exit(0 if entry["max_gpu_gb"] < 76 else f"smoke: {entry['max_gpu_gb']} GB peak is too close to 80 GB") +PY + log "smoke passed: $(tail -1 "$W/smoke.log")" +} + gpu() { local deadline deadline=$(date -u -d "+$((MAX_GPU_HOURS * 60 - 20)) minutes" +%FT%TZ) watchdog - - pip install -q -r "$HF/inference/requirements.txt" - pip freeze > "$W/pip-freeze-gpu.txt" - nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$W/pod.log" + gpu_setup sweep calibration @@ -139,11 +177,12 @@ all() { local fetch_pid=$! pack > "$W/pack.log" 2>&1 wait "$fetch_pid" || { log "fetch failed: see $W/fetch.log"; return 1; } + smoke gpu for keep in $PRUNE; do prune "$keep"; done log "all done: $(echo "$PRUNE" | sed "s|[0-9]*|$W/v41-reap&|g"). Pod left running for the Mac to pull; watchdog still armed." } -stage=${1:?usage: pod.sh fetch|pack|gpu|prune KEEP|all} +stage=${1:?usage: pod.sh fetch|pack|smoke|gpu|prune KEEP|all} shift "$stage" "$@" From d7d91e671060b1bd0ecb2219c7b32d65a46e6b54 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:22:51 -0800 Subject: [PATCH 23/74] docs: record the idle-GPU prep failure on the first H100 pod --- .../2026-11-05-v41-runpod-reap-runbook.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md index 82d3c27..59d7c06 100644 --- a/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md +++ b/docs/deepseek-v4-flash/2026-11-05-v41-runpod-reap-runbook.md @@ -191,6 +191,36 @@ stock. These are the lessons from that day. `npm uninstall -g --prefix @anthropic-ai/claude-code`, so that `claude` resolves to a working install. +## Failure: the H100 sat idle for hours on prep work (2026-11-06) + +The 1×H100 SXM pod `m13sexafpkqia6` ($3.99/h, EUR-NO-2) ran `pod.sh all`. +Its GPU stayed at 0% for more than 2 hours: + +- **19:14–20:10:** the 510 GB weight fetch and sha256 check, at about 160–210 MB/s. +- **19:14 to 21:12 and later:** `build_pack.py`. Most of that time it was + streaming all 35 GB (gzip) of `common-pile/stackexchange_filtered` at about + 7 MB/s, unauthenticated, to keep posts from five sites only. + +That is at least $9 of GPU time spent on work that needs no GPU. It also pushes +the sweeps toward the 8 h watchdog. **This should have been caught before the +pod was rented.** The single-GPU design copied "fetch and pack" from the +8×H200 single-pod mode as GPU-pod time, and nobody questioned that or timed +the pack. + +**Rules for the next run:** + +- **Do no prep on a GPU pod.** Build and validate the pack on the Mac or on a + cheap CPU pod first. It needs only the tokenizer and public data. Download + the weights to a network volume from a CPU pod. Then create the GPU pod on + that volume. Its first command is `pod.sh smoke`. +- **Time slow inputs on cheap hardware.** Before any GPU spend, measure every + dataset stream and download in the pack. +- **Don't stream a whole corpus to keep a slice of it.** For StackExchange, + read only the per-site files, or pre-filter once on a CPU and keep the + result. Set `HF_TOKEN` (Jack enters it himself) so streams aren't throttled. +- **Count idle GPU time in the cost estimate.** If prep has to share the pod, + say up front how many idle GPU-hours it costs. + ## Steps (network-volume mode) Only you can do the steps marked **(you)**. I do not enter credentials or From 1795112388e1b09af2ee0a18594a806593dec934 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:46:55 -0800 Subject: [PATCH 24/74] runpod: build_pack exits without finalization (datasets threads aborted it after a complete pack) --- scripts/runpod/v41_reap/build_pack.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/runpod/v41_reap/build_pack.py b/scripts/runpod/v41_reap/build_pack.py index 487d0d4..82e2a39 100644 --- a/scripts/runpod/v41_reap/build_pack.py +++ b/scripts/runpod/v41_reap/build_pack.py @@ -32,6 +32,7 @@ import hashlib import json import re +import os import sys import urllib.request from collections import Counter, defaultdict @@ -378,4 +379,10 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - raise SystemExit(main()) + status = main() + sys.stdout.flush() + # Streaming `datasets` leaves native worker threads behind, and interpreter + # finalization then aborts with PyGILState_Release (exit 134). That happened + # on 2026-11-06 after the pack was fully written, and it stopped the pod. + # Skip finalization: every output is already closed. + os._exit(status) From c56d7e1dcebc2c08e2619ef76899edad566d6aed Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:16:47 -0800 Subject: [PATCH 25/74] runpod: 262144-token chunks (524288 OOMed in the indexer on the H100) The hidden state carried between layers costs about 63 KB per resident token on the GPU. With 524288-token chunks it held about 65 GB, and the indexer's 16 GiB allocation for a 16K record ran out of memory. CHUNK_TOKENS is now a knob with a default of 262144. --- scripts/runpod/v41_reap/pod.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/runpod/v41_reap/pod.sh b/scripts/runpod/v41_reap/pod.sh index 8e13984..1d5e605 100755 --- a/scripts/runpod/v41_reap/pod.sh +++ b/scripts/runpod/v41_reap/pod.sh @@ -23,6 +23,9 @@ OUT=$W/out CANDIDATES=${CANDIDATES:-"256 192 128 96"} PRUNE=${PRUNE:-"192"} # REAP 50%. REAP 75% (96) after the Mac pulls 192: 1.2 TB holds one MAX_GPU_HOURS=${MAX_GPU_HOURS:-8} # hard stop for the GPU pod (1xH100 SXM secure $3.99/h) +# Tokens (x streams) resident per chunk. The hidden state carried between layers costs ~63 KB/token +# on the GPU: 524288 left ~65 GB in use and the 16K-record indexer's 16 GiB alloc OOMed (2026-11-07). +CHUNK_TOKENS=${CHUNK_TOKENS:-262144} export PYTHONPATH=$W/keep/src # ssh sessions miss the pod's env (RUNPOD_POD_ID, the pod-scoped RUNPOD_API_KEY): load it, never print it if [[ -z "${RUNPOD_POD_ID:-}" && -r /etc/rp_environment ]]; then source /etc/rp_environment; fi @@ -91,7 +94,7 @@ sweep() { # one collect.py per GPU, sessions sharded across them; extra args go log "sweep $name on $n GPU(s)" for ((i = 0; i < n; i++)); do CUDA_VISIBLE_DEVICES=$i python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/pack/v41-pack.jsonl" \ - --out "$OUT" --sweep "$name" --shard "$i/$n" --deadline-utc "$deadline" "$@" \ + --out "$OUT" --sweep "$name" --shard "$i/$n" --deadline-utc "$deadline" --chunk-tokens "$CHUNK_TOKENS" "$@" \ > "$W/sweep-$name-$i.log" 2>&1 & pids+=($!) done From cc570783388aa790b638659bdbf58ef4d598d98c Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:17:36 -0800 Subject: [PATCH 26/74] v41 recovery: local eval harness, KEEP refits, precision allocation, sidecars, routed tiers Local tooling for recovering the published DeepSeek-V4.1-Flash REAP checkpoints on the M4 Max: - local_eval.py scores router-masked variants against the pod's teacher logits, with KEEP code overrides, imatrix capture, residual sidecars, and per-token precision tiers (--dynamic, --tau) under a no-stall LRU residual cache with async SSD admission (--cache-k, --cache-warm). - keep_quant.py fits E8P 2-bit and E8 1-bit codes with pod or pruned-model importance, and nested residual codes on top of another run (--residual-of). - precision_alloc.py: static per-expert rung allocation under a unit budget. - sidecar_fit.py: sequential reduced-rank ridge sidecars fit against the teacher MoE, plus sidecar_pod.sh for a one-GPU pod run. - Recovery log with steps 1-2d: selection on pruned-model routing explains the static allocation gain; second-pass pruning and pruned-model importance add nothing. --- .../2026-11-08-v41-reap-recovery-log.md | 518 ++++++++++++++++++ .../2026-11-08-v41-reap75-local-test-plan.md | 142 +++++ scripts/runpod/v41_reap/keep_quant.py | 142 +++++ scripts/runpod/v41_reap/local_eval.py | 449 +++++++++++++++ scripts/runpod/v41_reap/local_parity.py | 140 +++++ scripts/runpod/v41_reap/precision_alloc.py | 101 ++++ scripts/runpod/v41_reap/requirements-ref.txt | 9 + scripts/runpod/v41_reap/sidecar_fit.py | 314 +++++++++++ scripts/runpod/v41_reap/sidecar_pod.sh | 23 + scripts/runpod/v41_reap/torch_kernel.py | 110 +++- tests/test_runpod_v41_routed_precision.py | 64 +++ tests/test_runpod_v41_torch_kernel.py | 96 ++++ 12 files changed, 2092 insertions(+), 16 deletions(-) create mode 100644 docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md create mode 100644 docs/deepseek-v4-flash/2026-11-08-v41-reap75-local-test-plan.md create mode 100644 scripts/runpod/v41_reap/keep_quant.py create mode 100644 scripts/runpod/v41_reap/local_eval.py create mode 100644 scripts/runpod/v41_reap/local_parity.py create mode 100644 scripts/runpod/v41_reap/precision_alloc.py create mode 100644 scripts/runpod/v41_reap/requirements-ref.txt create mode 100644 scripts/runpod/v41_reap/sidecar_fit.py create mode 100755 scripts/runpod/v41_reap/sidecar_pod.sh create mode 100644 tests/test_runpod_v41_routed_precision.py create mode 100644 tests/test_runpod_v41_torch_kernel.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md new file mode 100644 index 0000000..8e6f276 --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -0,0 +1,518 @@ +# DeepSeek-V4.1-Flash REAP recovery: progress log + +Started 2026-11-08. Branch `jack/hopeful-cori-iszsfq`. This log records what was run, what +it showed, and what was learned, in order. The plan is the five-step recovery list from the +same day (non-uniform budgets, KEEP importance on the pruned model, residual sidecars, router +KD, LoRA KD), applied to the published REAP checkpoints +([REAP-50](https://huggingface.co/jackmaxwil/DeepSeek-V4.1-Flash-REAP-50), +[REAP-75](https://huggingface.co/jackmaxwil/DeepSeek-V4.1-Flash-REAP-75)). + +Compute rules for this campaign: the Mac first. A Runpod pod only for a real experiment that +the Mac cannot do in reasonable time, never for setup or idle waiting. Mac memory cap 64 GB for +these runs (Jack, 2026-11-08). + +## Starting point + +REAP degradation, measured on the pod by router masking inside the full model (437,350 +held-out positions; KL over the teacher's top 2,048 logits): + +| Experts kept / 384 | KL | top-1 | NLL change | +|---:|---:|---:|---:| +| 256 | 0.511 | 0.842 | +0.025 | +| 192 (REAP-50) | 0.810 | 0.803 | -0.138 | +| 128 | 1.114 | 0.755 | -0.109 | +| 96 (REAP-75) | 1.349 | 0.720 | -0.131 | + +- Fit: KL ≈ 1.84 · p^1.17 for pruned fraction p (worst miss 0.03); 1 − top-1 ≈ 0.33 · p^0.69. + The marginal cost is ~0.005 KL per expert per layer, nearly flat from 33% to 75% pruned. +- Agentic and tool-use text loses the most and loses it early (exponent 0.85–1.1); business, + code-diffs, cyber, networking and chat lose late (exponent 2–2.5). +- KEEP's own quant loss on V4-Flash (unpruned, 2.031 bpw) was KL 0.149 / top-1 0.899. If the + two add, REAP-50 + KEEP ≈ 0.96 KL. KEEP's gate (KL ≤ 0.30, top-1 ≥ 0.85) fails on REAP alone + at every tested level, so REAP loss is the main target. +- Both exported checkpoints were run on the Mac (CPU, reference code, dequantize-then-matmul + stand-ins) over 5 eval sessions and matched the pod's masked-router numbers: REAP-75 KL 1.378 + vs 1.385, REAP-50 0.809 vs 0.841. Per-position differences of 0.1–0.3 KL are run-to-run + chaos from rounding; the same session moved 0.17 KL between a solo and a batched run. + +## Tooling + +- `scripts/runpod/v41_reap/torch_kernel.py`: exact CPU ports of the FP8/FP4 kernels (unit tests + in `tests/test_runpod_v41_torch_kernel.py`). +- `scripts/runpod/v41_reap/local_parity.py`: one exported checkpoint against the pod's numbers. +- `scripts/runpod/v41_reap/local_eval.py`: any number of router-masked variants of a local + checkpoint in one layer-streamed pass, scored per position against the pod teacher's top-2,048 + logits. Variants may keep a different count per layer. +- Mac eval set (`~/Models/v41-reap-run/mac-eval-sessions.json`): 2 eval sessions per domain + (22 sessions, 11 domains), each cut to its first 1,024 tokens, 22,506 positions. Cutting is + exact because the model is causal; a smoke run matched the pod's per-position values for the + same positions (REAP-50, eval-00070: 0.480 local vs 0.465 pod). Domains are equally weighted + here, unlike the pod's position-weighted "all", and only early context (≤1,024 tokens) is + covered. +- Cost: about 4.5 minutes fixed per pass over REAP-50's weights, plus per-token compute. + +## Step 1: non-uniform per-layer budgets + +Question: at a fixed total expert budget, does moving experts between layers beat REAP's +uniform count? + +Allocation rule tried first (`mass-*`): REAP order within each layer is kept, so every variant +is nested inside REAP-50. Across layers, experts are removed greedily by their share of the +layer's routed output mass (Σ g·‖f‖ over calibration, normalized per layer), lowest first, +with a floor per layer. Variants (`~/Models/v41-reap-run/alloc/`): + +| Variant | Mean kept | Per-layer range | Kept mass share (mean) | +|---|---:|---|---:| +| uniform-96 (REAP-75) | 96 | 96 | 0.650 | +| mass-96 | 96 | 24–180 | 0.700 | +| mass-96-f64 | 96 | 64–180 | 0.678 | +| uniform-128 | 128 | 128 | 0.725 | +| mass-128-f64 | 128 | 64–192 | 0.764 | + +A frequency-based rule (`freq-*`) was built and rejected before running: it would keep only +10% of the output mass in layers 0–1. + +Mass-based allocation cuts layers 20–31 and 36–39 hardest, where a few experts carry most of +the output. It treats every layer as equally sensitive, which is the assumption this run tests. + +Results (`~/Models/v41-reap-run/evals/step1`, 22 sessions × 1,024 tokens, one pass of six +variants, 8,211 s, peak 22.5 GB footprint): + +| Variant | KL (domain mean) | top-1 | NLL change | +|---|---:|---:|---:| +| REAP-50 (192) | 0.588 | 0.819 | -0.043 | +| uniform-128 | **0.955** | **0.754** | +0.160 | +| mass-128-f64 | 1.046 | 0.718 | +0.394 | +| uniform-96 (REAP-75) | **1.191** | **0.707** | +0.306 | +| mass-96-f64 | 1.344 | 0.673 | +0.584 | +| mass-96 | 1.457 | 0.636 | +0.815 | + +Per domain (KL): + +| Domain | REAP-50 | uni-128 | mass-128-f64 | uni-96 | mass-96-f64 | mass-96 | +|---|---:|---:|---:|---:|---:|---:| +| agentic (system prompt, see below) | 0.346 | 0.572 | 0.722 | 0.768 | 1.109 | 1.291 | +| bash | 0.265 | 0.507 | 0.599 | 0.671 | 0.888 | 1.053 | +| business | 0.711 | 1.020 | 1.304 | 1.385 | 1.511 | 1.724 | +| business-email | 0.413 | 0.711 | 0.972 | 0.992 | 1.108 | 1.356 | +| code-diffs | 0.263 | 0.461 | 0.625 | 0.647 | 0.813 | 1.049 | +| cyber | 0.566 | 1.049 | 1.414 | 1.518 | 1.953 | 2.060 | +| english-chat | 0.321 | 0.755 | 0.866 | 1.042 | 1.105 | 1.229 | +| english-prose | 0.359 | 0.604 | 0.706 | 0.773 | 0.894 | 0.945 | +| networking | 0.336 | 0.658 | 0.970 | 0.896 | 1.104 | 1.293 | +| tool-use | 2.544 | 3.595 | **2.609** | 3.645 | 3.195 | **2.740** | + +Learnings: + +- **Mass-based reallocation is worse than uniform**, by 0.09 KL at 128 and 0.15–0.27 at 96, on + every domain except tool-use. The layers it cuts hardest (20–31, 36–39) are more sensitive than + their output-mass share says, so "low mass share = cheap to cut" is false across layers. Within + a layer REAP's ranking holds; across layers, mass share is not a sensitivity measure. +- **Tool-use runs the other way.** Mass allocation cuts tool-use KL by ~1.0 at both budgets + (3.60 → 2.61 at 128, 3.65 → 2.74 at 96), even though it hurts everything else. Tool-use leans on + the experts uniform pruning removes from the early layers (mass keeps more there). Tool-use is + also the worst domain under every variant (KL 2.5 even at REAP-50), so a domain-aware budget + (or keeping tool-use experts in the early layers) is a candidate, not a global reallocation. +- The REAP-50 → uniform-128 → uniform-96 curve on the Mac (0.59 / 0.95 / 1.19) has the same + shape as the pod's (0.81 / 1.11 / 1.35). Mac numbers are lower because they cover only the first + 1,024 tokens and weight domains equally. +- **Eval-set flaw found:** all four agentic sessions (agentic-python and agentic-rust) share a + ~3,900-token system prompt, so at 1,024 tokens they score the same text and returned identical + numbers. The "agentic" row is the system prompt only, and the domain mean double-counts it. The + 108 eval sessions have only 100 distinct 1,024-token prefixes. A later pass that needs agentic + coverage must run those sessions past token ~3,900 (about 4× their cost). +- Next for step 1, if it is pursued: measure sensitivity directly (cut one block of layers at a + time and score) instead of inferring it from mass. Deprioritized behind steps 2–3, because the + best case here is a small shift around uniform, while tool-use is the outlier worth chasing. + +## Step 2: KEEP quantization of the pruned model (running) + +- `scripts/runpod/v41_reap/keep_quant.py` fits KEEP's importance-aware E8P codes to REAP-50's + 192 experts per layer from their FP4 weights (the numpy fit with the Metal search, which is + byte-identical to the shipped V4-Flash path). +- V4.1's down projection has input width 2304, which group size 512 does not divide, so down + uses group 256: 2.031 bpw for gate/up, 2.0625 for down, 2.042 averaged. +- Weighted relative MSE (pod importance, layer 0): gate/up 0.084, down 0.047; layer 1: 0.089 / + 0.061. About 500 s per layer while the step-1 eval shares the CPU, so ~5.5 h for 40 layers. +- `local_eval.py --expert-override` runs these codes in the eval: `fp4-bf16` (exact FP4 decode, + the control that isolates removing FP8 activation quantization), a code directory, or a + per-expert mix of two directories. +- Later layers fit slightly worse: layers 16–17 gate/up 0.092, down 0.082; about 350 s per layer + once the CPU eval ended. +- 2-bit fit done for all 40 layers (`quant/e8p-podimp`, about 4.5 h): mean weighted relative MSE + gate/up 0.090, down 0.074, worst layer 17 (0.092). The 1-bit fit started right after. +- The "new importance" half (imatrix measured on the pruned model, 96 calibration records of + 1,024 tokens, `--capture-imatrix`) started when step 1 finished (output + `~/Models/v41-reap-run/imatrix/reap50-pruned.npz`). +- The first capture (all 96 records in one pass) hit the 64 GB guard after 56 minutes: peak RSS + 64.4 GB, of which only 32 GB was the process's own footprint (the rest is mmapped checkpoint + pages, which RSS counts). The importance stats are sums, so it was rerun as four passes of 24 + records (`imatrix/chunk{0..3}.npz`) to be added together. +- Capture done: 76,471 tokens, four passes of 15–38 minutes at ~31.5 GB peak RSS, summed into + `imatrix/reap50-pruned.npz`. 2 of 7,680 surviving experts never routed; 29 routed fewer than + 20 times. +- Pruned-model importance vs the full model's (cosine per expert, mean over routed experts), + against a noise floor from splitting the pod's 24 sessions into two halves: + + | Space | Pod half vs half (noise) | Pruned vs pod | Worst layers | + |---|---:|---:|---| + | gate/up input (hidden) | 0.995 | 0.976 | 15, 21–27 (~0.96) | + | down input (expert intermediate) | 0.895 | 0.788 | 6–9 (~0.73) | + + Both shifts are larger than the noise, and the down input shifts much more. The comparison is + confounded: the Mac records are the first 1,024 tokens of each calibration session, while the + pod summed whole sessions, so data and pruning both differ. The full model cannot run on this + Mac to separate them, so the test is end to end: requantize with the pruned importance only if + the 2-bit codes with pod importance show a quant loss worth recovering. +- Eval started (`evals/step2-fp4-bf16`, then `evals/step2-e8p-podimp`): REAP-50 and uniform-96, + first with the exact FP4 weights in bf16 (control: isolates dropping FP8 activation + quantization), then with the 2-bit codes. + +### Result: KEEP 2-bit on the pruned model (pod importance) + +Same 22 sessions × 1,024 tokens. Each pass ~65 min (two variants), peak RSS 52–54 GB. + +| Experts | Weights | KL | top-1 | NLL change | +|---|---|---:|---:|---:| +| 192 (REAP-50) | FP4 + FP8 activations (shipped) | 0.588 | 0.819 | -0.043 | +| 192 | FP4 decoded to bf16 (control) | 0.578 | 0.821 | -0.028 | +| 192 | KEEP 2-bit, pod importance | **0.747** | **0.788** | -0.092 | +| 96 (REAP-75) | FP4 + FP8 activations | 1.191 | 0.707 | +0.306 | +| 96 | FP4 decoded to bf16 | 1.192 | 0.708 | +0.306 | +| 96 | KEEP 2-bit, pod importance | 1.216 | 0.689 | +0.313 | + +Per domain, REAP-50 (control → 2-bit): agentic 0.33 → 0.42, bash 0.26 → 0.34, business 0.71 → +0.75, business-email 0.42 → 0.48, code-diffs 0.26 → 0.35, cyber 0.55 → 0.78, english-chat 0.33 → +0.46, english-prose 0.36 → 0.41, networking 0.34 → 0.45, tool-use 2.47 → 3.36. + +Learnings: + +- The control matches the shipped path (0.578 vs 0.588), so the override path is sound and FP8 + activation quantization costs nothing measurable here. +- **KEEP 2-bit adds +0.17 KL / −0.033 top-1 on REAP-50**, close to its +0.149 on unpruned + V4-Flash. On REAP-50, REAP and KEEP losses roughly add. +- **On REAP-75 it adds only +0.02 KL**, with the same per-expert codes. The losses are far from + additive once the prune error is large: KL against the teacher is already high, and extra weight + noise moves the distribution in directions that matter less. Two consequences: the quant loss is + worth recovering only at mild prune levels, and the step-2b comparisons (all at high KL) will + favour whatever has less prune error. +- Tool-use takes the biggest quant hit at REAP-50 (+0.89 KL) and even improves slightly at REAP-75 + (3.65 → 3.48). Tool-use is the fragile domain under every change so far. + +Next (queued behind the 1-bit fit): requantize at 2 bits with the pruned-model importance +(`quant/e8p-prunedimp`, blended: pruned importance per expert where it routed ≥20 times, the pod's +otherwise, each normalized to mean 1, floored at 1e-4; 29 experts fall back), on the GPU alongside +the step-2b evals on the CPU. + +## Step 2b: is REAP the right tool, or a low-bit floor for every expert? (Jack's idea) + +Idea: pruning is KEEP's precision ladder with a hard 0-bit rung. Give every expert a low-bit +base instead, and let the runtime raise precision per expert (nested residual stages, hot +residuals in RAM, the rest on SSD). + +Memory facts: one expert slot across all 40 layers costs 0.359 GB at 2.031 bpw, so all 384 +experts are ~138 GB at 2 bpw (does not fit) and ~70 GB at 1.03 bpw (fits). Decode reads 240 +experts per token (6 × 40 layers), so speed depends on bits read, not bits stored. + +### Residency replay (done, CPU, no model) + +Recorded full-model routing from the 24 calibration sessions (49,152 tokens, first 2,048 +tokens each) replayed through a per-layer hot set of 1-bit residual stages (4.6 MB per expert +per layer), resetting at each session: + +| Hot set / layer | Residual RAM | Decayed-count miss rate | Misses / token | Gate mass on misses (LRU, 6 sessions) | +|---:|---:|---:|---:|---:| +| 48 | 8.8 GB | 26.5% | 63.5 | 27.8% | +| 96 | 17.7 GB | 13.6% | 32.7 | 14.5% | +| 128 | 23.6 GB | 9.2% | 22.0 | 8.8% | +| 192 | 35.3 GB | 4.4% | 10.5 | 3.7% | +| 256 | 47.1 GB | 2.0% | 4.9 | 1.7% | + +- Fetching every miss from the SSD (~0.7 ms per 4.6 MB at 6–7 GB/s) costs ~23 ms/token at 96 + and ~7 ms at 192, against a ~32 ms budget at 31 tok/s. Fetch-on-miss only works at 192+ hot, + i.e. 70 GB base + 35 GB residuals, which does not leave room on a 128 GB machine. +- The workable form is serve-from-base: a miss runs at the 1-bit base stage with no stall, and + the residual is promoted in the background. At hot 128 (23.6 GB) about 9% of the routed gate + mass would run at base precision. +- LRU and decayed counts behave alike; a static REAP-ordered hot set is far worse (58% misses at + 128). REAP ranks by mean contribution per routed token, not by frequency. +- Side finding: under the full model's routing, REAP-50's 192 survivors receive only ~60% of + routed slots (the static-192 miss rate is 40%). REAP-50 reroutes 40% of all expert slots. + +### Static frontier test (queued) + +Inside REAP-50, REAP-75's 96 experts per layer are "hot" and the other 96 "cold": + +| Variant | Memory (relative) | Question | +|---|---:|---| +| hot 96 @ 2.03, cold dropped (REAP-75 + KEEP) | ~195 | prune baseline | +| all 192 @ 1.03 | ~198 | REAP vs a low-bit floor at equal memory | +| hot 96 @ 2.03, cold 96 @ 1.03 | ~294 | the cost of a cold expert at 1 bit vs 0 | + +KEEP's 1-bit tier is the E8 codebook with 8-bit indices (`--code-bits 8`): 1.031 bpw gate/up, +1.0625 down. Smoke fit on layer 0: weighted relative MSE 0.32 against 0.094 at 2 bits, so the +1-bit floor is coarse at the weight level; the KL test decides. + +The 1-bit fit finished in 3.3 h (`quant/e8-1bit-podimp`, much faster than the smoke estimate once +the CPU was free): mean weighted relative MSE gate/up 0.311, down 0.253, about 3.4× the 2-bit +error. Evals running: `evals/step2b-1bit` (REAP-50 and uniform-96, all experts at 1 bit), then +`evals/step2b-mixed` (REAP-75's 96 at 2 bits, the other 96 at 1 bit). The 2-bit row of the table +is `evals/step2-e8p-podimp` uniform-96 (KL 1.216). + +## Step 2c: precision as a learned lever (static allocation first) + +Framing (Jack, 2026-11-08): precision should be a routed, per-token dimension of the MoE, with +nested codes, so that pruning (the 0-bit rung), quantization and offloading become one +optimization: minimize E[KL] + λ·E[bits read per token] subject to bits stored ≤ RAM. Static +decisions are how many rungs each expert stores; dynamic decisions are which rung each routed +expert runs at for a token. + +Evidence from this campaign that per-axis budgets are the wrong shape: + +- Losses do not add: the same 2-bit codes cost +0.17 KL on REAP-50 and +0.02 on REAP-75. +- Proxies fail: moving experts between layers by mass share lost to uniform (step 1). +- Allocation is input-dependent: the reallocation that hurt every domain helped tool-use by ~1 KL. +- Decode reads 240 experts per token, so bits read (bandwidth) and bits stored (RAM) are separate + levers. + +Closest prior work found so far (not a full literature search): HOBBIT (arXiv 2411.01433) serves +cache-miss experts from low-precision copies with layer-level prefetch (up to 9.93× decode on +consumer GPUs); Mixtral offloading, Pre-gated MoE, Fate and ProMoE predict next-layer experts; +Matryoshka-style quantization trains nested codes; HAWQ / Bayesian Bits learn per-layer +bit-widths. The combination tested here, where the router itself chooses (expert, rung) under a +bits penalty with nested codes, has not been found in that set. + +Test ladder: + +1. Static allocation without training (Mac). `scripts/runpod/v41_reap/precision_alloc.py`: + within each layer, rungs 0/1/2 bits under a fixed unit budget (1-bit expert = 1 unit), greedy + by estimated error reduction per unit, cost = expert's routed output-mass share × relative + output error at that rung (layer-mean KEEP errors; dropped = 1). Uniform budget across layers. +2. Check the estimator against measured KL (step 2b points). +3. Router over (expert, rung) trained by KD with a bits penalty (pod; merges with step 4). +4. Trainable nested codebooks (pod; merges with step 5). + +Step 1 allocations (`~/Models/v41-reap-run/alloc/prec-{192,288}`): + +| Budget / layer | Memory equivalent | Kept (mean) | at 2 bits | at 1 bit | Overlap of 2-bit set with REAP-75 | +|---:|---|---:|---:|---:|---:| +| 192 | REAP-75 at 2 bits | 101 | 91 | 10 | | +| 288 | REAP-75 at 2 bits + rest at 1 bit | 147 | 141 | 6 | 54% | + +The estimator does not favour a 1-bit floor. Routed mass is concentrated enough that upgrading a +heavy expert to 2 bits buys more estimated error than keeping a light one at 1 bit, so it drops +the light tail and keeps almost everything else at 2 bits. Its 2-bit set also differs from REAP's +(54% overlap), because it ranks by total routed mass (frequency × contribution) where REAP ranks by +contribution per routed token. Step 2b's "all 192 at 1 bit" eval tests the floor directly; the +step-2c evals (`evals/step2c-prec-{192,288}`) are queued behind it. If the floor beats these, +mass × error is the same kind of failed proxy as in step 1 (it ignores rerouting and rare but +critical experts), and allocation has to be learned against KL. + +### SSD churn under admission thresholds (residency replay, CPU) + +Same 24 calibration sessions (49,152 tokens of full-model routing), decayed-count hot set per +layer, 4.6 MB per residual stage. A miss whose gate weight is below the threshold is served at the +base rung and never promoted; misses at or above it are read from the SSD. Gate weights sum to 1.5 +per token-layer (route_scale); median 0.22. Output `~/Models/v41-reap-run/residency-admission.txt`. + +| Hot / layer | Threshold | Promotions / token | SSD MB / token | GB/s at 30 tok/s | Gate mass never promoted | +|---:|---:|---:|---:|---:|---:| +| 96 | none | 33.0 | 152 | 4.55 | 0 | +| 96 | 0.2 | 15.2 | 70 | 2.09 | 5.7% | +| 128 | none | 22.6 | 104 | 3.12 | 0 | +| 128 | 0.2 | 10.4 | 48 | 1.44 | 4.4% | +| 128 | 0.3 | 3.4 | 16 | 0.47 | 16.5% | +| 192 | none | 11.8 | 54 | 1.63 | 0 | +| 192 | 0.2 | 6.2 | 29 | 0.86 | 3.2% | +| 192 | 0.3 | 3.0 | 14 | 0.42 | 15.9% | + +- Promoting every miss is not sustainable below 192 hot: 3–4.5 GB/s of continuous reads against + ~5.4 GB/s sequential, before any prefill traffic. +- A gate-weight threshold of 0.2 halves the traffic at every hot size for 3–6% of gate mass left + at base precision. At hot 128 (23.6 GB of residuals) that is 1.4 GB/s, about a quarter of the + SSD. 0.3 cuts traffic another 3× but leaves ~16% of the mass unpromoted. +- The gate weight is a usable admission signal, and it is the same quantity a learned router + would weigh against a bits penalty: low-weight picks contribute little, so they are the cheap + ones to run at base. Whether 4–6% of gate mass at 1 bit is affordable is what step 2b measures + (all 192 at 1 bit is the 100% case). +- Not yet replayed: next-layer prefetch (needs hidden states, not just routes) and second-hit + admission. + +### Step 2b results: the static frontier + +Same 22 sessions × 1,024 tokens. Units are per-layer memory in 1-bit-expert equivalents +(2-bit expert = 2, FP4 ≈ 4.25). + +| Configuration (inside REAP-50) | Units / layer | KL | top-1 | NLL change | +|---|---:|---:|---:|---:| +| REAP-75's 96 at 2 bits, rest dropped | 192 | **1.216** | **0.689** | +0.313 | +| all 192 at 1 bit | 192 | 1.358 | 0.637 | +0.425 | +| REAP-75's 96 at 1 bit, rest dropped | 96 | 1.826 | 0.565 | +0.943 | +| REAP-75's 96 at 2 bits, other 96 at 1 bit | 288 | 0.954 | 0.738 | +0.010 | +| all 192 at 2 bits | 384 | 0.747 | 0.788 | -0.092 | +| all 192 FP4 (bf16 decode) | ~816 | 0.578 | 0.821 | -0.028 | + +Per domain (KL), all-192 at 1 bit / mixed 288: agentic 1.01 / 0.55, bash 0.82 / 0.45, business +1.80 / 1.06, business-email 1.16 / 0.75, code-diffs 0.93 / 0.47, cyber 1.57 / 1.09, english-chat +1.59 / 0.87, english-prose 0.90 / 0.56, networking 0.93 / 0.60, tool-use 3.22 / 3.56. + +Learnings: + +- **At equal memory, pruning plus 2 bits beats a 1-bit floor**: 1.216 vs 1.358 KL. The static + version of "no REAP, every expert at low bits" loses. The step-2c estimator predicted this + direction (drop the light tail, keep the rest at 2 bits). +- **KEEP's 1-bit tier is expensive**: all of REAP-50 at 1 bit costs +0.78 KL over FP4, against + +0.17 at 2 bits. The 1-bit E8 codebook (weight error 0.31) is too coarse to be a floor that runs + a large share of the routed mass. +- **But a 1-bit rung beats dropping**: adding REAP-50's other 96 experts at 1 bit to REAP-75 at 2 + bits cuts KL from 1.216 to 0.954. Upgrading those same 96 from 1 to 2 bits cuts it further to + 0.747. Per unit of memory the two steps are worth about the same (-0.26 and -0.21 KL per 96 + units), and upgrading REAP-75's 96 from 1 to 2 bits is worth more (-0.61). Marginal value per + unit: hot 1→2 > cold 0→1 ≈ cold 1→2. That is the shape a learned allocator should find, and it + is consistent with the mass × error estimator. +- Frontier so far (units, KL): (96, 1.83), (192, 1.22), (288, 0.95), (384, 0.75), (~816, 0.58). +- For the serve-from-base runtime: running a miss at the 1-bit base is not free. The all-1-bit + pass is the 100%-of-mass case (+0.78 KL); at the 4% never-promoted mass of threshold 0.2 the + cost is unmeasured and should not be assumed linear. A better low rung (1.5 bits, or a 1-bit + base with a small learned correction) would change this picture more than any allocation rule. +- **Tool-use anomaly**: tool-use KL is 2.5–3.6 under every variant and does not order with + memory (1-bit uniform-96 at 3.06 beats 2-bit uniform-96 at 3.48), and its NLL change on REAP-50 + is -1.1 to -1.5 (the pruned model predicts the actual tokens much better than the teacher). That + pattern looks like a measurement problem in those two sessions' teacher logits (alignment or + template), not a model effect. To check before tool-use numbers drive any decision. + +### Tool-use anomaly: checked + +- Teacher logits are aligned with the tokens (argmax matches the next token 56–83% of the time, + the current token 0%). The pod's own REAP-50 run shows the same gap as the Mac, so it is not a + local bug. +- On the pod data the pruned model beats the full model on actual-token NLL in the tool and code + domains: tool-use -1.22 nats, agentic -0.33, english-chat -0.27, bash -0.16; it loses on prose, + business and cyber (+0.3 to +0.4). On tool-use session eval-00018 the full model's NLL is 3.9–5.2 + nats per token in every position band, against 1.4–4.3 for REAP-50. +- The full model has catastrophic positions: 1.4% of all eval positions have teacher NLL > 15 + nats (3.6% in agentic), on trivially predictable tokens (for example `"title": "` then + `Content`, teacher 21.8 nats, REAP-50 0.0). REAP-50 has them too, but fewer (2,199 vs 2,895). +- Those positions carry only ~4% of the measured KL; dropping them moves the domain means by + 0.015–0.02 (REAP-50 0.578 → 0.563, 2-bit 0.747 → 0.725, 1-bit 1.358 → 1.335). The frontier + conclusions do not change. Tool-use KL stays ~2.4 without them, so the tool-use gap is broad, + not just spikes. +- Reading: on DSML tool-call text the full model, run through the reference torch path with the + FP4/FP8 kernels, is far worse than a frontier model should be, and removing experts improves it. + Either some experts misbehave in this numeric path on that text, or the full model really is + weak there. Until that is settled, tool-use KL against this teacher is not a quality measure + (the teacher is the worse model on that domain), and NLL on actual tokens should be reported + next to it. Isolating it needs the full model (pod or base download), so it is noted, not + chased. + +### Step 2c results: static per-expert allocation + +| Configuration | Units / layer | KL | top-1 | NLL change | +|---|---:|---:|---:|---:| +| REAP-75's 96 at 2 bits (prune + KEEP) | 192 | 1.216 | 0.689 | +0.313 | +| all 192 at 1 bit | 192 | 1.358 | 0.637 | +0.425 | +| **prec-192** (≈101 kept: 91 at 2 bits, 10 at 1 bit) | 192 | **1.014** | **0.732** | +0.158 | +| REAP-75's 96 at 2 bits + other 96 at 1 bit | 288 | 0.954 | 0.738 | +0.010 | +| **prec-288** (≈147 kept: 141 at 2 bits, 6 at 1 bit) | 288 | **0.809** | **0.773** | -0.039 | +| all 192 at 2 bits | 384 | 0.747 | 0.788 | -0.092 | + +Per domain (KL), REAP-75@2 → prec-192: agentic 0.73 → 0.55, bash 0.71 → 0.52, business 1.51 → +1.38, business-email 1.07 → 0.78, code-diffs 0.72 → 0.50, cyber 1.53 → 1.09, english-chat 1.19 → +1.19, english-prose 0.82 → 0.67, networking 0.90 → 0.68, tool-use 3.48 → 3.25. + +Learnings: + +- **At REAP-75's memory, the allocator cuts KL by 0.20 (−17%) and lifts top-1 by 4.3 points**, + better or equal on every domain. At 288 units it beats the REAP-ordered mix by 0.145 and comes + within 0.06 of all-192-at-2-bits with 75% of the memory. +- The estimator is a crude proxy (layer-mean errors, mass × error, no training), so this is a + floor on what a learned allocation can do, not a ceiling. +- Open question before crediting "precision allocation": prec-192 is mostly a different choice + of which ~96 experts get 2 bits (58% overlap with REAP-75), plus a small 1-bit tail. Two things + changed at once: the ranking criterion (total routed mass instead of REAP's contribution per + routed token) and the routing it is measured on (the pruned model's own instead of the full + model's). Ablation running (`evals/step2c-selection`, all 96 at 2 bits): full-model total mass, + pruned-model REAP score, pruned-model total mass. + +### Step 2c selection ablation + +All variants keep 96 experts per layer at KEEP 2 bits (`e8p-podimp`, 192 units), so only the +choice of experts differs. Domain-mean KL over the same held-out eval set: + +| 96 experts chosen by | Overlap with REAP-75 | KL | top-1 | +|---|---:|---:|---:| +| REAP score, full-model routing (shipped REAP-75) | 1.000 | 1.216 | 0.689 | +| total routed mass, full-model routing | 0.664 | 1.191 | 0.707 | +| REAP score, pruned-model routing | 0.784 | 1.188 | 0.701 | +| **total routed mass, pruned-model routing** | 0.580 | **1.039** | **0.733** | +| prec-192 (the same ranking, plus a 1-bit tail) | — | 1.014 | 0.732 | + +Learnings: + +- **Selection is most of the step-2c gain.** Of prec-192's 0.202 KL cut, 0.177 comes from + choosing the 96 experts by total routed mass measured on REAP-50's own routing; the 1-bit tail + adds 0.025 (plus 0.023 on tool-use, the domain where it helps most). +- **The two changes only pay together.** Each alone moves KL by about 0.025; combined they move + it by 0.177. Ranking by total mass on the full model's routing does not see how routing + redistributes once half the experts are gone, and REAP's per-route score on the pruned model + still favors experts that are strong when picked but rarely picked. +- So a second prune pass measured on the first pruned model (a cheap form of iterative pruning) + is worth more than KEEP's precision trade at this budget. The 1-bit rung is still a real but + small positive at 192 units, and grows at 288 (prec-288 0.809 vs mixed 0.954, of which the + ranking accounts for an unmeasured share). +- The mass capture came from calibration records (first 1,024 tokens of the 24 calibration + sessions), not the eval set, so this is not leakage. + +### Step 2 closed: pruned-model importance for the KEEP fit + +The 2-bit refit with importance measured on REAP-50 itself (`quant/e8p-prunedimp`) was evaluated +against the pod-importance fit (`e8p-podimp`), same eval set: + +| Variant | Importance | KL | top-1 | NLL change | +|---|---|---:|---:|---:| +| REAP-50, all 192 at 2 bits | pod (full model) | 0.747 | 0.788 | -0.092 | +| REAP-50, all 192 at 2 bits | pruned model | 0.753 | 0.790 | -0.106 | +| prunedmass-96 at 2 bits | pod (full model) | 1.039 | 0.733 | +0.154 | +| prunedmass-96 at 2 bits | pruned model | 1.027 | 0.734 | +0.159 | + +Learnings: + +- **No effect.** The differences (+0.006, -0.012) are within run-to-run noise, and the + prunedmass-96 gain sits almost entirely in tool-use (3.42 to 3.28), the domain whose KL is not + a quality measure against this teacher. Without tool-use the two fits tie. +- So the input importance KEEP weights its fit by barely moves after pruning (hidden-space cosine + 0.976 in step 2 already said so), while the routing mass used for selection moves a lot (step 2c + ablation). Recomputing statistics on the pruned model matters for which experts to keep, not + for how to quantize them. +- Peak RSS 56.6 GB, 48 minutes for two variants. + +### Step 2d: iterative pruning (192 → 144 → 96) + +First pass: the top 144 experts per layer by total routed mass on REAP-50's routing +(`alloc/iter/pm-144.json`). Routing recaptured on REAP-50 masked to those 144 (same 96 +calibration records in 4 chunks, 13–15 minutes each, about 31 GB peak RSS, +`imatrix/pm144-chunk*.npz`), then the top 96 of the 144 by the new mass (`alloc/iter/iter-96.json`). +The second pass changes little: iter-96 overlaps the one-pass 96 by 97.3% (minimum 94.8% in any +layer, about 2.6 experts per layer). Eval at KEEP 2 bits (`e8p-podimp`), `evals/step2d-iterative`: + +| Configuration | Units / layer | KL | KL without tool-use | top-1 | +|---|---:|---:|---:|---:| +| one-pass mass, 96 at 2 bits | 192 | 1.039 | 0.801 | 0.733 | +| **two-pass mass, 96 at 2 bits** | 192 | 1.016 | 0.795 | 0.731 | +| prec-192 (one-pass ranking + 1-bit tail) | 192 | 1.014 | — | 0.732 | +| REAP-75's 96 at 2 bits + other 96 at 1 bit | 288 | 0.954 | 0.693 | 0.738 | +| **one-pass mass, 144 at 2 bits** | 288 | 0.812 | 0.566 | 0.770 | +| prec-288 (≈141 at 2 bits + 6 at 1 bit) | 288 | 0.809 | 0.564 | 0.773 | + +Learnings: + +- **Iterative pruning saturates after one pass.** The second pass gains 0.023 KL, below the + 0.03 bar set before the run, and 0.006 of it outside tool-use. One re-measurement on the pruned + model captures nearly all of the routing redistribution. +- **At 288 units the 1-bit rung adds nothing either:** 144 experts at 2 bits score 0.812 against + prec-288's 0.809. prec-288's lead over the REAP-ordered mix is all selection. +- Net for step 2: under the current codes, the whole gain over REAP-75 + KEEP comes from + re-ranking experts by total routed mass on the pruned model's routing. The static 1-bit rung + helps only marginally (0.025 at 192 units, nothing at 288). A low rung that pays needs to be + better than this 1-bit E8 fit (errors 0.31/0.25), or dynamic, or both. +- Step 2 is done; step 3 (residual sidecars) is next. diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap75-local-test-plan.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap75-local-test-plan.md new file mode 100644 index 0000000..eece40e --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap75-local-test-plan.md @@ -0,0 +1,142 @@ +# DeepSeek-V4.1-Flash-REAP-75: local test plan (M4 Max) + +Goal: run the published [jackmaxwil/DeepSeek-V4.1-Flash-REAP-75](https://huggingface.co/jackmaxwil/DeepSeek-V4.1-Flash-REAP-75) +on this Mac. First prove that the exported checkpoint computes what the Runpod eval measured. Then +produce text from it. Speed is a later question. + +REAP-75 is the testable one. REAP-50's routed experts alone are 148 GiB, more than the machine's +128 GiB, so they cannot stay in memory. + +## What is in the checkpoint + +| Part | Size | How a local run touches it | +|---|---:|---| +| Engram n-gram tables (layers 1 and 14) | 189 GiB | sparse row gathers from the shard files on disk (memory-mapped), never loaded whole | +| Routed experts, 96 per layer, FP4 | 81 GiB | streamed one layer at a time | +| Vision tower | 2 GiB | not used | +| Attention, shared experts, embeddings, head | 1 GiB | per layer | + +Disk is no longer a constraint: 948 GiB was free on 2026-11-08, after the cleanup. + +## Why the torch reference path comes first, not MLX + +V4.1 is not V4-Flash plus Engram. On top of Engram it adds: +- cross-layer KV sharing (`kv_source_layers`); +- a two-level indexer (`index_source_layers`, candidate blocks); +- per-layer compression ratios; +- DSpark heads. + +The MLX runtime in `src/keep/quality/dsv4_*` and `src/keep/convert/dsv4_resident.py` covers none +of this. + +The single-GPU driver already runs DeepSeek's own `inference/model.py` unchanged, one layer at a +time: +- `scripts/runpod/v41_reap/stream.py` already gathers Engram rows from the shards on the host. +- `scripts/runpod/v41_reap/torch_kernel.py` already ports `sparse_attn` and `hc_split_sinkhorn` to + plain torch. + +The only pieces missing on a Mac are the quantized GEMMs and the activation quantizers. Those are +the tilelang kernels, which are GPU-only. + +So the first local run reuses the code that produced the published numbers. An MLX port is worth +it only once that run passes, and only if the speed matters. + +## Steps + +1. **Environment.** + - What: a separate venv with `torch` (CPU/MPS wheel), `safetensors`, `tokenizers`, `sympy`, + `numpy` and `huggingface_hub`. The repo `.venv` has no torch, and should stay that way. + - Check: `python -c "import torch; torch.float8_e8m0fnu"` works. + +2. **Download REAP-75.** + - What: `hf download jackmaxwil/DeepSeek-V4.1-Flash-REAP-75 --local-dir ~/Models/v41-reap75`, then + `shasum -a 256 -c SHA256SUMS` in that directory. That is 273 GiB; time depends on the line. + - Check: every checksum passes. + +3. **Pull the eval pack.** + - What: copy only `pack/v41-pack.jsonl` and `pack/manifest.json` from the stopped pod + `neckuzwv11dy2i`, probably tens of MB. Start the pod with zero GPUs for this and stop it right + after. + - Why: the stats dataset has teacher logits but no input token ids, and the parity check needs + the exact eval inputs. + - After: terminate the pod, with Jack's OK. + - Check: `pack_sha256` in the manifest matches `run-config.json` (`6a354705…`). + +4. **CPU stand-ins for the quantized kernels**, in `torch_kernel.py`: + - `act_quant`: an exact port of the per-32-block FP8 activation quantization (`fp8_block_size = 32` in this release), including the + e8m0 scale rounding. It quantizes and dequantizes, so the numerics match the pod. + - `fp8_gemm`: dequantize the e4m3 weight with its 32x32 e8m0 block scales, then do a bf16/fp32 + matmul. + - `fp4_gemm`: decode the e2m1 nibbles (low nibble first, as in `keep.convert.fp4_expert`) with + per-32 e8m0 scales, then matmul. Dequantize one expert at a time, so a layer never holds more + than one expert in bf16. + - Check: extend `tests/test_runpod_v41_stream.py` with a toy model in fp8 and fp4. It must match + the same model computed in plain bf16 from the dequantized weights to within float tolerance. + +5. **Memory gate.** + - What: run one real layer on one short eval record, under `/usr/bin/time -l`. + - Expected peak: a few GiB of weights plus activations. + - Check: the projected peak for the full run is written down. If it is over 24 GB, stop and ask + Jack. + +6. **Parity: the real test.** + - What: run the exported REAP-75 layer-streamed over 2 or 3 of the shortest eval sessions. Score + it against the teacher top-K in the stats dataset (`eval/sessions/*.npz`). + - Compare with the pod's masked-router numbers for the same positions (`student-96/`). + - Expectation: KL, NLL and top-1 agree per position up to the small difference between the + tilelang kernels and the dequantize-then-matmul stand-ins. + - Pass: |ΔKL| and |ΔNLL| averaged per session are small next to the REAP-75 numbers themselves + (KL 1.35, NLL change −0.13 on all), and top-1 agreement is within a point. + - If it passes, the "not run end to end" caveat comes off both model cards. + - If it fails, the export (`prune.py`: router slicing, expert renumbering) is wrong. Then fix the + export and republish both models. + +7. **Text smoke test.** + - What: greedy decode of about 20 tokens from a short coding prompt, layer-streamed. + - Cost: each token re-reads about 84 GiB of non-Engram weights from the SSD, so expect tens of + seconds per token. That is fine for a smoke test. + - Option: keeping the experts in memory (about 85 GiB) would be faster, but it is over the 24 GB + budget and needs Jack's OK. + +## Speed path after parity: Rapid-MLX + +[raullenchai/Rapid-MLX](https://github.com/raullenchai/Rapid-MLX) (read at `7d2b20c`, 2026-11-08) already +runs DeepSeek-V4.1-Flash-1017 in MLX, from the same source revision (`1bae0e10`). What matters for +REAP-75: + +- **Engram:** tables are re-quantized to MLX affine 4-bit, group 32 (about 61 GB per layer). Rows + are read from mmap with MADV_RANDOM, through a 16,384-row LRU and a one-layer-ahead prefetch + (`rapid_mlx/models/deepseek_v41_native/engram.py`). +- **Experts:** read from `config.json` (`n_routed_experts`, optional per-layer counts), so a + pruned checkpoint fits the model code. +- **Conversion:** their loader reads only converted MLX builds. `convert.py` dequantizes our + FP8/FP4 release format and requantizes it. It asserts expert ids are contiguous from 0, which + `prune.py` already guarantees. But it buffers a whole quantized Engram table (about 61 GB) before + saving, over the 24 GB budget, so it would need a streaming write first. +- **Fit (estimated from their byte counts):** with their 2/3-bit GPTQ recipe, REAP-75 is about + 58 GB resident plus 8 GB of DSpark drafter, with Engram on the SSD. That is over the 24 GB budget + and needs Jack's OK. +- **Kernel gap at 96 experts:** their fused decode-MoE kernel requires `dim < 16 * n_experts` + (`moe_decode.py:557`). At 96 experts that fails (5120 >= 1536), so it silently falls back to the + stock path. +- **Wired memory:** `load.py:181-188` always raises the MLX wired limit, which conflicts with this + repo's wired-memory policy. Their product lane also refuses machines under 224 GiB + (`artifacts.py:57`). +- **Speed:** 34-49 tok/s greedy with DSpark speculative decoding, on an M3 Ultra 256 GB. +- **Correctness evidence:** teacher-forced KL 0.26 and top-1 85% against the BF16 source. + +Never run their scripts or package from the checkout. `scripts/calibrate_deepseek_v41_reap.py` +executes `runtime/runtime.py` from a model directory. + +## Out of scope for now + +- An MLX port of V4.1 (Engram, KV sharing, indexer, compression). +- The vision path and DSpark/MTP speculative heads. +- REAP-50, which has to be tested on a GPU pod or with the experts streamed. +- Speed work. + +## Decisions for Jack + +- Go for the 273 GiB download (step 2). +- Starting the stopped pod for a few minutes with zero GPUs to pull the pack, then terminating it + (step 3). diff --git a/scripts/runpod/v41_reap/keep_quant.py b/scripts/runpod/v41_reap/keep_quant.py new file mode 100644 index 0000000..38c2a62 --- /dev/null +++ b/scripts/runpod/v41_reap/keep_quant.py @@ -0,0 +1,142 @@ +"""KEEP E8P quantization of a local REAP checkpoint's routed experts, one layer at a time. + +The source is the checkpoint's FP4 experts, decoded exactly. Each projection is fitted with +KEEP's importance-aware E8P fit (``fit_projection``: 16-bit codes, weighted least-squares group +scales), with a per-input-channel importance diagonal: + +- ``--importance pod``: the full model's calibration imatrix from the Runpod run + (``calibration/sessions/cal-*.npz``, summed), indexed by each survivor's original expert id. + This is what KEEP would use without re-calibrating after the prune. +- ``--importance PATH``: an imatrix npz measured on the pruned model itself (``local_calib.py``), + already in the checkpoint's local expert order. + +gate/up (w1/w3, input 5120) use group size 512 (2.031 bpw); down (w2, input 2304, which 512 +does not divide) uses 256 (2.0625 bpw). ``--code-bits 8`` uses KEEP's 1-bit E8 codebook at the +same group sizes instead (1.031 / 1.0625 bpw). ``--residual-of DIR`` fits the residual of another +run's codes (``W - decode(DIR)``) instead of ``W``: a nested code, whose base is usable alone and +base + residual is the higher-precision tier; ``total_wrel_mse`` scores the sum against ``W``. Output: one ``layer-XX.npz`` per layer with +``{w1,w2,w3}_codes`` uint16 ``[E, out, in/8]`` and ``{w1,w2,w3}_scales`` fp16 ``[E, out, in/g]``, +plus ``metrics.jsonl``. Finished layers are skipped, so the run can resume. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import numpy as np + +KIT = Path(__file__).resolve().parent +sys.path.insert(0, str(KIT)) +sys.path.insert(0, str(KIT.parents[2] / "src")) + +PROJ_SPACE = {"w1": "hidden", "w3": "hidden", "w2": "down"} + + +def pod_importance(stats: Path) -> dict[str, np.ndarray]: + """Summed ``importance_sum__{hidden,down}`` [L, 384, in] over the calibration sessions.""" + + total: dict[str, np.ndarray] = {} + for path in sorted((stats / "calibration" / "sessions").glob("cal-*.npz")): + with np.load(path) as z: + for space in ("hidden", "down"): + a = z[f"importance_sum__{space}"].astype(np.float64) + total[space] = total.get(space, 0) + a + assert total, "no calibration sessions found" + return total + + +def decode_fp4(weight, scale) -> np.ndarray: + import torch_kernel + + w = torch_kernel.decode_e2m1(weight) # [out, in] fp32 + return (w.unflatten(-1, (-1, 32)) * scale.float()[..., None]).flatten(-2).numpy() + + +def decode_codes(z, proj: str, e: int) -> np.ndarray: + """One expert's projection from a keep_quant layer file, as ``local_eval.expert_override`` decodes it.""" + + from keep.vq.e8 import e8_1bit_grid, e8p_full_grid + + grid = (e8p_full_grid() if int(z["code_bits"]) == 16 else e8_1bit_grid()).astype(np.float32) + vec = grid[z[f"{proj}_codes"][e].astype(np.int64)] # [out, in/8, 8] + scales = z[f"{proj}_scales"][e].astype(np.float32) + vec = vec * np.repeat(scales, vec.shape[1] // scales.shape[-1], axis=-1)[..., None] + return vec.reshape(vec.shape[0], -1) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--hf-path", required=True, type=Path) + p.add_argument("--checkpoint-survivors", required=True, type=Path) + p.add_argument("--stats", type=Path, help="local REAP-stats copy (for --importance pod)") + p.add_argument("--importance", required=True, help="'pod' or a local imatrix npz") + p.add_argument("--out", required=True, type=Path) + p.add_argument("--layers", default="all", help="'all' or comma-separated layer ids") + p.add_argument("--iterations", type=int, default=8) + p.add_argument("--backend", default="metal", help="E8P search backend: metal or numpy") + p.add_argument("--code-bits", type=int, default=16, choices=(8, 16), + help="16: E8P, 2 bits/weight + scales; 8: E8 1-bit codebook, 1 bit/weight + scales") + p.add_argument("--experts", type=int, default=0, help="debug: fit only the first N experts per layer") + p.add_argument("--residual-of", type=Path, help="fit W minus this keep_quant run's decoded codes (nested tier)") + args = p.parse_args(argv) + + import stream + from keep.convert.dsv4_vq_pilot import fit_projection, projection_error, reconstruct_quantized + + survivors = json.loads(args.checkpoint_survivors.read_text())["survivors"] + ckpt = stream.HFCheckpoint(args.hf_path) + n_layers = len(survivors) + layers = range(n_layers) if args.layers == "all" else [int(x) for x in args.layers.split(",")] + if args.importance == "pod": + full = pod_importance(args.stats) + imp = {space: np.stack([full[space][l][sorted(survivors[str(l)])] for l in range(n_layers)]) for space in full} + else: + with np.load(args.importance) as z: + imp = {space: z[f"importance__{space}"].astype(np.float64) for space in ("hidden", "down")} + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "config.json").write_text(json.dumps(vars(args) | {"hf_path": str(args.hf_path), "out": str(args.out), + "checkpoint_survivors": str(args.checkpoint_survivors), + "stats": str(args.stats)}, indent=2, default=str)) + + for layer in layers: + dest = args.out / f"layer-{layer:02d}.npz" + if dest.exists(): + continue + t0 = time.time() + sd = ckpt.load(f"layers.{layer}.ffn.experts.") + n_exp = len(survivors[str(layer)]) if not args.experts else args.experts + arrays: dict[str, list[np.ndarray]] = {} + errs = {k: [] for k in PROJ_SPACE} + total = {k: [] for k in PROJ_SPACE} + base = np.load(args.residual_of / f"layer-{layer:02d}.npz") if args.residual_of else None + for e in range(n_exp): + for proj, space in PROJ_SPACE.items(): + w = decode_fp4(sd[f"{e}.{proj}.weight"], sd[f"{e}.{proj}.scale"]) + group = 512 if w.shape[1] % 512 == 0 else 256 + d = imp[space][layer, e].astype(np.float32) + b = decode_codes(base, proj, e) if base is not None else 0.0 + q, metrics, _ = fit_projection(w - b, d, group_size=group, + code_bits=args.code_bits, iterations=args.iterations, backend=args.backend) + if base is not None: + total[proj].append(projection_error(w, b + reconstruct_quantized(q), d).weighted_relative_mse) + arrays.setdefault(f"{proj}_codes", []).append(np.asarray(q.codes, dtype=np.uint16)) + arrays.setdefault(f"{proj}_scales", []).append(np.asarray(q.scales, dtype=np.float16)) + errs[proj].append(metrics.weighted_relative_mse if hasattr(metrics, "weighted_relative_mse") else float("nan")) + tmp = dest.with_suffix(".tmp.npz") + np.savez(tmp, code_bits=np.asarray(args.code_bits), **{k: np.stack(v) for k, v in arrays.items()}) + tmp.rename(dest) + row = {"layer": layer, "experts": n_exp, "seconds": round(time.time() - t0, 1), + **{f"{k}_wrel_mse": float(np.mean(v)) for k, v in errs.items()}, + **{f"{k}_total_wrel_mse": float(np.mean(v)) for k, v in total.items() if v}} + with (args.out / "metrics.jsonl").open("a") as f: + f.write(json.dumps(row) + "\n") + print(json.dumps(row), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py new file mode 100644 index 0000000..208f037 --- /dev/null +++ b/scripts/runpod/v41_reap/local_eval.py @@ -0,0 +1,449 @@ +"""Score router-masked variants of a local REAP checkpoint against the pod's teacher logits. + +Every variant keeps a subset of the checkpoint's experts in each layer (any count per layer, +so non-uniform budgets work) by setting the dropped experts' gate bias to -inf, which is what +``collect.py --survivors`` did on the pod and equals physically slicing them out. All variants +go through one layer-streamed pass on CPU, through DeepSeek's reference code with the +``torch_kernel`` stand-ins. + +Sessions are cut to their first ``--max-tokens`` tokens. The model is causal, so the logits at +the kept positions are the ones the pod computed on the whole record; the teacher's top-K rows +for those positions are compared directly. + + python local_eval.py --hf-path ~/Models/v41-reap50 \\ + --checkpoint-survivors ~/Models/v41-reap-run/stats/survivors-192.json \\ + --pack ~/Models/v41-reap-run/pack/v41-pack.jsonl --stats ~/Models/v41-reap-run/stats \\ + --sessions-file ~/Models/v41-reap-run/mac-eval-sessions.json --max-tokens 1024 \\ + --variant reap50=all --variant reap75=~/Models/v41-reap-run/stats/survivors-96.json \\ + --out ~/Models/v41-reap-run/evals/baseline + +A variant spec is ``all`` (the checkpoint as is) or a survivors JSON in the base model's expert +ids (``{"survivors": {"": [ids...]}}``), each layer a subset of the checkpoint's. + +``--expert-override`` swaps every routed expert's weights for bf16 ones before the pass: +``fp4-bf16`` decodes the checkpoint's FP4 weights exactly (the control), a directory of +``keep_quant.py`` layer files decodes KEEP's codes (E8P or 1-bit E8), and +``HOT_DIR@SET.json+REST_DIR`` takes the experts in a survivors set from one directory and the +rest from another (a per-expert precision tier). Overridden experts run as plain bf16 +GEMMs without FP8 activation quantization, as KEEP's runtime does. + +``--capture-imatrix OUT.npz`` measures KEEP's importance diagonal on this checkpoint instead of +scoring: one variant, any sessions (``name`` or ``name#record``, calibration records allowed). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import resource +import sys +import time +from pathlib import Path + +import numpy as np + +KIT = Path(__file__).resolve().parent +sys.path.insert(0, str(KIT)) +sys.path.insert(0, str(KIT.parents[2] / "src")) + +from local_parity import rss_guard # noqa: E402 + + +def local_masks(spec: str, ckpt_survivors: dict[str, list[int]]) -> dict[int, np.ndarray] | None: + """Per layer, a bool array over the checkpoint's local experts: True = dropped.""" + + if spec == "all": + return None + wanted = json.loads(Path(spec).expanduser().read_text())["survivors"] + masks = {} + for layer, have in ckpt_survivors.items(): + have = sorted(have) + keep = set(wanted[layer]) + assert keep <= set(have), f"layer {layer}: variant keeps experts the checkpoint dropped" + masks[int(layer)] = np.asarray([e not in keep for e in have]) + return masks + + +def expert_override(spec: str, n_experts: int, ckpt_survivors: dict): + """``layer -> {(expert, proj): bf16 [out, in]}`` from FP4 (``fp4-bf16``) or E8P codes.""" + + import torch + + import torch_kernel + + if spec == "fp4-bf16": + def from_state(layer: int, sd: dict) -> dict: + out = {} + for e in range(n_experts): + for proj in ("w1", "w2", "w3"): + w = torch_kernel.decode_e2m1(sd[f"ffn.experts.{e}.{proj}.weight"]) + s = sd[f"ffn.experts.{e}.{proj}.scale"].float() + out[(e, proj)] = (w.unflatten(-1, (-1, 32)) * s[..., None]).flatten(-2).bfloat16() + return out + return from_state + + from keep.vq.e8 import e8_1bit_grid, e8p_full_grid + + grids = {16: torch.as_tensor(e8p_full_grid().astype(np.float32)), 8: torch.as_tensor(e8_1bit_grid().astype(np.float32))} + + def from_dir(root: Path): + def from_codes(layer: int, sd: dict, only: set[int] | None = None) -> dict: + out = {} + with np.load(root / f"layer-{layer:02d}.npz") as z: + grid = grids[int(z["code_bits"]) if "code_bits" in z.files else 16] + for proj in ("w1", "w2", "w3"): + codes = torch.as_tensor(z[f"{proj}_codes"].astype(np.int64)) # [E, out, in/8] + scales = torch.as_tensor(z[f"{proj}_scales"].astype(np.float32)) # [E, out, in/g] + assert codes.shape[0] == n_experts, (layer, proj, codes.shape) + for e in range(n_experts): + if only is not None and e not in only: + continue + vec = grid[codes[e]] # [out, in/8, 8] + words = vec.shape[1] // scales.shape[-1] + vec = vec * scales[e].repeat_interleave(words, dim=-1)[..., None] + out[(e, proj)] = vec.flatten(-2).bfloat16() + return out + return from_codes + + # "HOT_DIR@HOT_SET.json+REST_DIR": the hot set's experts from one source, the rest from another + if "@" in spec: + hot_spec, _, rest = spec.partition("@") + hot_set_path, _, rest_spec = rest.partition("+") + hot = from_dir(Path(hot_spec).expanduser()) + other = from_dir(Path(rest_spec).expanduser()) + hot_ids = json.loads(Path(hot_set_path).expanduser().read_text())["survivors"] + + def mixed(layer: int, sd: dict) -> dict: + have = sorted(ckpt_survivors[str(layer)]) + is_hot = {i for i, e in enumerate(have) if e in set(hot_ids[str(layer)])} + out = hot(layer, sd, is_hot) + out.update(other(layer, sd, set(range(n_experts)) - is_hot)) # decode each expert once + return out + return mixed + + return from_dir(Path(spec).expanduser()) + + +def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: list[int], + warm: np.ndarray | None, counts: np.ndarray) -> np.ndarray: + """Which (token, slot) pairs may run base + residual, under an LRU of ``k`` resident residuals. + + ``ind``/``wts``: [T, top_k] routed experts and gate weights, the records of ``bounds`` back to + back. A slot runs high precision only if its expert's residual is already resident. After each + token, every slot with gate >= ``tau`` admits (or refreshes) its residual: an async SSD fetch + that serves the next token on, so decode never stalls. Each record starts from ``warm`` (or + empty). ``counts[e, 4]`` accumulates admissions, i.e. residual fetches.""" + + from collections import OrderedDict + + hi = np.zeros(ind.shape, dtype=bool) + start = 0 + for n in bounds: + lru = OrderedDict((int(e), None) for e in (warm if warm is not None else [])) + for t in range(start, start + n): + row = ind[t].tolist() + hi[t] = [e in lru for e in row] + for s, e in enumerate(row): + if wts[t, s] >= tau: + if e in lru: + lru.move_to_end(e) + else: + counts[e, 4] += 1 + lru[e] = None + if len(lru) > k: + lru.popitem(last=False) + start += n + return hi + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--hf-path", required=True, type=Path) + p.add_argument("--checkpoint-survivors", required=True, type=Path, help="the survivors JSON the checkpoint was exported from") + p.add_argument("--pack", required=True, type=Path) + p.add_argument("--stats", required=True, type=Path) + p.add_argument("--sessions-file", required=True, type=Path) + p.add_argument("--max-tokens", type=int, default=1024) + p.add_argument("--variant", action="append", required=True, metavar="NAME=SPEC") + p.add_argument("--out", required=True, type=Path) + p.add_argument("--max-rss-gb", type=float, default=64.0) + p.add_argument("--threads", type=int, default=os.cpu_count()) + p.add_argument("--expert-override", help="'fp4-bf16' or a keep_quant.py output directory") + p.add_argument("--capture-imatrix", type=Path, help="measure the importance diagonal instead of scoring") + p.add_argument("--dynamic", help="BASE_DIR+RESIDUAL_DIR: per-token tiers, base alone or base + nested residual") + p.add_argument("--tau", action="append", default=[], metavar="NAME=VALUE", + help="with --dynamic, per variant: a slot runs base + residual when its gate weight >= VALUE") + p.add_argument("--cache-k", action="append", default=[], metavar="NAME=K", + help="with --dynamic: hi tier only from an LRU of K resident residuals per layer, admitted at gate >= tau") + p.add_argument("--cache-warm", type=Path, help="imatrix npz (reap__gate_norm_sum): warm-start each cache with the top-K") + p.add_argument("--sidecar", type=Path, help="a sidecar_fit.py output directory, added to every variant's MoE output") + p.add_argument("--sidecar-rank", type=int, help="use only the top-k sidecar components (stored largest first)") + args = p.parse_args(argv) + rss_guard(args.max_rss_gb) + + import torch + from transformers import AutoTokenizer + + import collect + import stream + import torch_kernel + + prune = json.loads((args.hf_path / "keep-prune.json").read_text()) + sha = hashlib.sha256(args.checkpoint_survivors.read_bytes()).hexdigest() + assert sha == prune["survivors_sha256"], "--checkpoint-survivors is not what this checkpoint was exported from" + ckpt_survivors = json.loads(args.checkpoint_survivors.read_text())["survivors"] + variants = {} + for spec in args.variant: + name, _, path = spec.partition("=") + masks = local_masks(path, ckpt_survivors) + variants[name] = None if masks is None else {l: torch.as_tensor(m) for l, m in masks.items()} + + torch_kernel.exact_quant = True + torch.set_num_threads(args.threads) + ref_dir = args.hf_path / "inference" + ref = stream.import_reference(ref_dir, kernel=torch_kernel) + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(33377335) + model_args = ref.ModelArgs(**json.loads((ref_dir / "config.json").read_text())) + model_args.max_batch_size = 1 + + sessions = collect.load_sessions(args.pack) + names = json.loads(args.sessions_file.read_text()) + records = {} + for name in names: + session, _, idx = name.partition("#") + recs = sessions[session] + if not idx: + assert len(recs) == 1, f"{session} has {len(recs)} records; name one as {session}#N" + record = recs[int(idx or 0)] + record = dict(record, token_ids=record["token_ids"][: args.max_tokens]) + records[name] = record + model_args.max_seq_len = max(len(r["token_ids"]) for r in records.values()) + + ckpt = stream.HFCheckpoint(args.hf_path) + layout = ref.EngramLayout.from_args(model_args) + hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) + engram_rows = {l: stream.EngramRows.from_checkpoint(ckpt, l, ref.fp8_block_size) for l in model_args.engram_layer_ids} + streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, "cpu") + state = collect.install_hooks(streamer, top_k=2048, rows_per_slice=256) + n_local = len(next(iter(ckpt_survivors.values()))) + + dyn_counts: dict[str, np.ndarray] = {} + if args.dynamic: # per-token precision: each (token, expert) slot picks its tier from its gate weight + import torch.nn.functional as F + + from keep.vq.e8 import e8_1bit_grid, e8p_full_grid + + base_dir, _, res_dir = args.dynamic.partition("+") + base_dir, res_dir = Path(base_dir).expanduser(), Path(res_dir).expanduser() + taus = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau)} + assert set(taus) == set(variants), "--tau needs one NAME=VALUE per --variant" + np_grids = {16: e8p_full_grid().astype(np.float32), 8: e8_1bit_grid().astype(np.float32)} + cache: dict = {"layer": None} + n_layers_ = int(model_args.n_layers) + cache_k = {name: int(k) for name, _, k in (s.partition("=") for s in args.cache_k)} + warm = None + if args.cache_warm: # warm start: each layer's top-K experts by routed mass (the static hot set) + warm = np.argsort(-np.load(args.cache_warm)["reap__gate_norm_sum"], axis=1, kind="stable") + for v in variants: # [layer, expert, (lo slots, hi slots, lo gate mass, hi gate mass, admissions)] + dyn_counts[v] = np.zeros((n_layers_, n_local, 5)) + + def simulate_cache(layer, ind, wts, k, tau, counts): + return residual_cache(ind, wts, k, tau, state["bounds"], None if warm is None else warm[layer, :k], counts) + + def tier_weights(layer: int, e: int) -> tuple[dict, dict]: + """(base, base + residual) bf16 weights of local expert ``e``, decoded once per layer.""" + if cache["layer"] != layer: + cache.clear() + cache["layer"] = layer + cache["z"] = [dict(np.load(d / f"layer-{layer:02d}.npz")) for d in (base_dir, res_dir)] + if e not in cache: + tiers = [] + for z in cache["z"]: + grid = np_grids[int(z["code_bits"])] + tier = {} + for proj in ("w1", "w2", "w3"): + vec = grid[z[f"{proj}_codes"][e].astype(np.int64)] + s = z[f"{proj}_scales"][e].astype(np.float32) + vec = vec * np.repeat(s, vec.shape[1] // s.shape[-1], axis=-1)[..., None] + tier[proj] = vec.reshape(vec.shape[0], -1) + tiers.append(tier) + lo = {p: torch.as_tensor(w).bfloat16() for p, w in tiers[0].items()} + hi = {p: torch.as_tensor(tiers[0][p] + tiers[1][p]).bfloat16() for p in tiers[0]} + cache[e] = (lo, hi) + return cache[e] + + def run_expert(x, w, tier, limit): + gate, up = F.linear(x, tier["w1"]).float(), F.linear(x, tier["w3"]).float() + if limit > 0: + up, gate = up.clamp(-limit, limit), gate.clamp(max=limit) + return F.linear((w * (F.silu(gate) * up)).to(x.dtype), tier["w2"]) + + def dynamic_forward(self, x, image_mask=None): + shape, layer = x.size(), self.layer_id + x = x.view(-1, self.dim) + weights, indices = self.gate(x, None) + tau, counts = taus[state["variant"]], dyn_counts[state["variant"]][layer] + y = torch.zeros_like(x, dtype=torch.float32) + limit = self.shared_experts.swiglu_limit + k = cache_k.get(state["variant"]) + hi_all = None if k is None else torch.as_tensor( + simulate_cache(layer, indices.numpy(), weights.float().numpy(), k, tau, counts)) + for i in torch.unique(indices).tolist(): + idx, top = torch.where(indices == i) + w = weights[idx, top] + hi_mask = w >= tau if hi_all is None else hi_all[idx, top] + lo_w, hi_w = tier_weights(layer, i) + for mask, tier, col in ((~hi_mask, lo_w, 0), (hi_mask, hi_w, 1)): + if mask.any(): + y[idx[mask]] += run_expert(x[idx[mask]], w[mask, None], tier, limit).float() + counts[i, col] += int(mask.sum()) + counts[i, col + 2] += float(w[mask].sum()) + y += self.shared_experts(x).float() + return y.type_as(x).view(shape) + + ref.MoE.forward = dynamic_forward + + if args.sidecar: # sidecar_fit.py output: y += ([x, 1] @ U) @ Vt on every MoE call + sidecars = {} + for layer in range(int(model_args.n_layers)): + with np.load(args.sidecar / f"layer-{layer:02d}.npz") as z: + U, Vt = z["U"][:, : args.sidecar_rank or None], z["Vt"][: args.sidecar_rank or None] + sidecars[layer] = (torch.as_tensor(U), torch.as_tensor(Vt)) + moe_forward = ref.MoE.forward + + def with_sidecar(self, x, image_mask=None): + y = moe_forward(self, x, image_mask) + U, Vt = sidecars[self.layer_id] + xf = x.reshape(-1, x.size(-1)).float() + corr = (xf @ U[:-1] + U[-1]) @ Vt + return (y.reshape(-1, y.size(-1)).float() + corr).type_as(y).view(y.shape) + + ref.MoE.forward = with_sidecar + + if args.expert_override: + decode = expert_override(args.expert_override, n_local, ckpt_survivors) + decoded: dict[int, dict] = {} + load_layer, build_block = streamer.load_layer, streamer.build_block + + def load_with_override(prefix, skip=()): # runs in the prefetch thread + sd = load_layer(prefix, skip) + if prefix.startswith("layers."): + layer = int(prefix.split(".")[1]) + decoded[layer] = decode(layer, sd) + return sd + + def build_with_override(layer, sd): + block = build_block(layer, sd) + for (e, proj), w in decoded.pop(layer).items(): + lin = getattr(block.ffn.experts[e], proj) + lin.weight = torch.nn.Parameter(w, requires_grad=False) + if hasattr(lin, "scale"): + del lin.scale + return block + + streamer.load_layer, streamer.build_block = load_with_override, build_with_override + + capture = None + if args.capture_imatrix: + assert len(variants) == 1, "--capture-imatrix takes one variant" + capture = collect.SessionStats(torch=torch, n_layers=int(model_args.n_layers), n_experts=n_local, + dims={"hidden": int(model_args.dim), "down": int(model_args.moe_inter_dim)}, device="cpu") + + state["bounds"] = [len(r["token_ids"]) for r in records.values()] # one MoE call holds a variant's records in order + items = { + (v, n): stream.Item(tokens=torch.as_tensor([r["token_ids"]], dtype=torch.long), group=(v, None)) + for v in variants + for n, r in records.items() + } + + def before_moe(layer: int, block, key) -> None: + bias = block.ffn.gate.bias + if not hasattr(block, "full_bias"): + block.full_bias = bias.data.clone() + bias.data.copy_(block.full_bias) + masks = variants[key[0]] + if masks is not None: + bias.data[masks[layer]] = float("-inf") + state["stats"] = capture + state["variant"] = key[0] + + t0 = time.time() + with torch.inference_mode(): + streamer.forward(list(items.values()), before_moe) + forward_s = time.time() - t0 + state["stats"] = None + + if capture is not None: + arrays = capture.arrays(list(range(int(model_args.n_layers)))) + args.capture_imatrix.parent.mkdir(parents=True, exist_ok=True) + np.savez(args.capture_imatrix, + importance__hidden=arrays["importance_sum__hidden"], importance__down=arrays["importance_sum__down"], + route_count=arrays["route_count__hidden"], reap__gate_norm_sum=arrays["reap__gate_norm_sum"], + reap__route_count=arrays["reap__route_count"], + sessions=np.asarray(names), tokens=np.asarray(sum(len(r["token_ids"]) for r in records.values()))) + print(json.dumps({"forward_seconds": round(forward_s, 1), "tokens": int(sum(len(r["token_ids"]) for r in records.values())), + "peak_rss_gb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 2**30, 2)}), flush=True) + return 0 + + args.out.mkdir(parents=True, exist_ok=True) + summary: dict = {"forward_seconds": round(forward_s, 1), "max_tokens": args.max_tokens, "variants": {}} + for name, record in records.items(): + teacher = np.load(args.stats / "eval" / "sessions" / f"{name}.npz") + positions = collect.target_positions(record) + rows = slice(0, positions.size) + assert np.array_equal(positions, teacher["positions"][rows]), name + targets = np.asarray(record["token_ids"], dtype=np.int64)[positions + 1] + teacher_nll = (teacher["logsumexp"][rows] - teacher["target_logit"][rows]).astype(np.float64) + state["positions"] = torch.as_tensor(positions) + state["targets"] = torch.as_tensor(targets) + state["teacher"] = { + "ids": torch.as_tensor(teacher["topk_logit_ids"][rows].astype(np.int64)), + "logprob": torch.as_tensor(teacher["topk_logit_values"][rows].astype(np.float32) - teacher["logsumexp"][rows, None]), + "targets": state["targets"], + } + for v in variants: + state["captured"] = {k: [] for k in collect.CAPTURE_KEYS} + with torch.inference_mode(): + streamer.head(streamer.final_hidden(items[(v, name)])) + cap = {k: np.concatenate(x) for k, x in state["captured"].items() if x} + out = { + "positions": positions.astype(np.int32), + "kl": cap["kl_on_teacher_topk"], + "nll_delta": (cap["student_nll"] - teacher_nll).astype(np.float32), + "top1": cap["top1_agree"], + } + (args.out / v).mkdir(exist_ok=True) + np.savez(args.out / v / f"{name}.npz", **out) + s = summary["variants"].setdefault(v, {}) + s[name] = {"domain": record["domain"], "positions": int(positions.size), + **{k: float(np.mean(out[k])) for k in ("kl", "nll_delta", "top1")}} + state["teacher"] = state["positions"] = state["targets"] = None + + for v, per in summary["variants"].items(): + doms: dict[str, list] = {} + for row in per.values(): + doms.setdefault(row["domain"], []).append(row) + by_dom = {d: {k: float(np.average([r[k] for r in rs], weights=[r["positions"] for r in rs])) for k in ("kl", "nll_delta", "top1")} + for d, rs in sorted(doms.items())} + per["_domains"] = by_dom + per["_domain_mean"] = {k: float(np.mean([x[k] for x in by_dom.values()])) for k in ("kl", "nll_delta", "top1")} + print(v, json.dumps(per["_domain_mean"]), flush=True) + for v, c in dyn_counts.items(): # per-token precision: how often (and with how much gate mass) the hi tier ran + np.save(args.out / v / "tier-counts.npy", c) + tot = c.sum(axis=(0, 1)) + summary["variants"][v]["_tiers"] = {"tau": taus[v], "cache_k": cache_k.get(v), + "hi_slot_share": float(tot[1] / tot[:2].sum()), + "hi_mass_share": float(tot[3] / tot[2:4].sum()), + "admissions_per_token_layer": float(tot[4] / (tot[:2].sum() / int(model_args.n_activated_experts)) / int(model_args.n_layers))} + print(v, json.dumps(summary["variants"][v]["_tiers"]), flush=True) + summary["peak_rss_gb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 2**30, 2) + (args.out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps({k: summary[k] for k in ("forward_seconds", "peak_rss_gb")}), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/local_parity.py b/scripts/runpod/v41_reap/local_parity.py new file mode 100644 index 0000000..e5343b5 --- /dev/null +++ b/scripts/runpod/v41_reap/local_parity.py @@ -0,0 +1,140 @@ +"""Run an exported REAP checkpoint on CPU and score it against the pod's teacher, per position. + +The Runpod eval scored each candidate by masking pruned experts out of the full model's +router (``collect.py --survivors``). This script runs the exported checkpoint itself, which +has those experts physically removed and the router sliced, through the same layer-streamed +reference code with the CPU kernel stand-ins in ``torch_kernel.py``. It reads the teacher's +top-K logits from the pod's eval npz files, so the full model is never needed here, then +compares KL, NLL and top-1 agreement with the pod's masked-router numbers for the same +positions. + + python local_parity.py --hf-path ~/Models/v41-reap75 \\ + --pack ~/Models/v41-reap-run/pack/v41-pack.jsonl \\ + --stats ~/Models/v41-reap-run/stats --keep 96 \\ + --sessions eval-00054 eval-00100 --out ~/Models/v41-reap-run/parity-96.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import resource +import sys +import threading +import time +from pathlib import Path + +import numpy as np + +KIT = Path(__file__).resolve().parent +sys.path.insert(0, str(KIT)) + + +def rss_guard(limit_gb: float) -> None: + """Exit hard if this process's peak resident size passes ``limit_gb`` (Jack's memory budget).""" + + def watch() -> None: + while True: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 2**30 # bytes on macOS + if peak > limit_gb: + print(f"peak RSS {peak:.1f} GB passed the {limit_gb} GB budget; exiting", flush=True) + os._exit(3) + time.sleep(2) + + threading.Thread(target=watch, daemon=True).start() + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--hf-path", required=True, type=Path, help="the exported REAP checkpoint") + p.add_argument("--pack", required=True, type=Path) + p.add_argument("--stats", required=True, type=Path, help="local copy of the REAP-stats dataset (eval/ and student-KEEP/)") + p.add_argument("--keep", required=True, help="the pod candidate to compare with, e.g. 96") + p.add_argument("--sessions", nargs="+", required=True) + p.add_argument("--out", required=True, type=Path) + p.add_argument("--max-rss-gb", type=float, default=24.0) + p.add_argument("--threads", type=int, default=os.cpu_count()) + args = p.parse_args(argv) + rss_guard(args.max_rss_gb) + + import torch + from transformers import AutoTokenizer + + import collect + import stream + import torch_kernel + + torch_kernel.exact_quant = True + torch.set_num_threads(args.threads) + ref_dir = args.hf_path / "inference" + ref = stream.import_reference(ref_dir, kernel=torch_kernel) + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(33377335) + model_args = ref.ModelArgs(**json.loads((ref_dir / "config.json").read_text())) + model_args.max_batch_size = 1 + + sessions = collect.load_sessions(args.pack) + records = {name: sessions[name][0] for name in args.sessions} + model_args.max_seq_len = max(len(r["token_ids"]) for r in records.values()) + + ckpt = stream.HFCheckpoint(args.hf_path) + layout = ref.EngramLayout.from_args(model_args) + hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) + engram_rows = {l: stream.EngramRows.from_checkpoint(ckpt, l, ref.fp8_block_size) for l in model_args.engram_layer_ids} + streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, "cpu") + state = collect.install_hooks(streamer, top_k=2048, rows_per_slice=256) + + items = {name: stream.Item(tokens=torch.as_tensor([r["token_ids"]], dtype=torch.long), group=("student", None)) # one MoE pass for all + for name, r in records.items()} + t0 = time.time() + with torch.inference_mode(): + streamer.forward(list(items.values())) + forward_s = time.time() - t0 + + report: dict[str, dict[str, float]] = {} + for name, record in records.items(): + teacher = np.load(args.stats / "eval" / "sessions" / f"{name}.npz") + pod = np.load(args.stats / f"student-{args.keep}" / "eval" / "sessions" / f"{name}.npz") + positions = collect.target_positions(record) + assert np.array_equal(positions, teacher["positions"]) and np.array_equal(positions, pod["positions"]), name + targets = np.asarray(record["token_ids"], dtype=np.int64)[positions + 1] + state["positions"] = torch.as_tensor(positions) + state["targets"] = torch.as_tensor(targets) + state["teacher"] = { + "ids": torch.as_tensor(teacher["topk_logit_ids"].astype(np.int64)), + "logprob": torch.as_tensor(teacher["topk_logit_values"].astype(np.float32) - teacher["logsumexp"][:, None]), + "targets": state["targets"], + } + state["captured"] = {k: [] for k in collect.CAPTURE_KEYS} + with torch.inference_mode(): + streamer.head(streamer.final_hidden(items[name])) + cap = {k: np.concatenate(v) for k, v in state["captured"].items() if v} + teacher_nll = teacher["logsumexp"] - teacher["target_logit"] + row = {"domain": record["domain"], "positions": int(positions.size)} + for key, local, remote in ( + ("kl", cap["kl_on_teacher_topk"], pod["kl_on_teacher_topk"]), + ("nll_delta", cap["student_nll"] - teacher_nll, pod["student_nll"] - teacher_nll), + ("top1_agree", cap["top1_agree"].astype(np.float64), pod["top1_agree"].astype(np.float64)), + ): + row[f"{key}_local"] = float(np.mean(local)) + row[f"{key}_pod"] = float(np.mean(remote)) + row[f"{key}_mean_abs_diff"] = float(np.mean(np.abs(local - remote))) + report[name] = row + print(json.dumps({name: row}), flush=True) + + summary = { + "hf_path": str(args.hf_path), + "keep": args.keep, + "forward_seconds": round(forward_s, 1), + "peak_rss_gb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 2**30, 2), + "sessions": report, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps({k: v for k, v in summary.items() if k != "sessions"}), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/precision_alloc.py b/scripts/runpod/v41_reap/precision_alloc.py new file mode 100644 index 0000000..d1fea57 --- /dev/null +++ b/scripts/runpod/v41_reap/precision_alloc.py @@ -0,0 +1,101 @@ +"""Static per-expert precision allocation inside REAP-50 (step 2c): rungs 0 (dropped), 1 and 2 bits. + +Within each layer, under a budget of B units (1-bit expert = 1 unit, 2-bit = 2), greedily buy the +upgrade with the largest estimated error reduction per unit: + + cost(e, r) = m_e * eps_r m_e = expert e's share of the layer's routed output mass + (sum of g * ||f|| on the pruned model, ``--capture-imatrix``) + eps_r = relative output error at rung r (eps_0 = 1, dropped) + +eps_1 and eps_2 are the layer means of KEEP's weighted relative MSE (gate/up averaged, plus down) +from ``keep_quant.py``'s ``metrics.jsonl``. Every layer gets the same budget, because moving +budget between layers by mass share lost to uniform in step 1. + +Writes ``keep.json`` (experts at rung >= 1, original ids, for ``local_eval.py --variant``) and +``hot.json`` (rung 2, for ``--expert-override HOT@hot.json+REST``), plus ``alloc.json``. +""" + +from __future__ import annotations + +import argparse +import heapq +import json +from pathlib import Path + +import numpy as np + + +def layer_eps(metrics: Path) -> dict[int, float]: + rows = [json.loads(line) for line in metrics.read_text().splitlines() if line.strip()] + return {r["layer"]: (r["w1_wrel_mse"] + r["w3_wrel_mse"]) / 2 + r["w2_wrel_mse"] for r in rows} + + +def allocate(mass: np.ndarray, eps: tuple[float, float, float], budget: int) -> np.ndarray: + """Rung per expert (0/1/2) maximizing the estimated error reduction under ``budget`` units.""" + + rung = np.zeros(len(mass), dtype=int) + heap = [(-m * (eps[0] - eps[1]), e) for e, m in enumerate(mass)] + heapq.heapify(heap) + left = budget + while heap and left > 0: + _, e = heapq.heappop(heap) + rung[e] += 1 + left -= 1 + if rung[e] == 1: + heapq.heappush(heap, (-mass[e] * (eps[1] - eps[2]), e)) + return rung + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--checkpoint-survivors", required=True, type=Path) + p.add_argument("--imatrix", required=True, type=Path, help="pruned-model capture npz (reap__gate_norm_sum)") + p.add_argument("--metrics-1bit", required=True, type=Path) + p.add_argument("--metrics-2bit", required=True, type=Path) + p.add_argument("--budget", required=True, type=int, help="units per layer (1-bit expert = 1, 2-bit = 2)") + p.add_argument("--out", required=True, type=Path) + args = p.parse_args(argv) + + survivors = json.loads(args.checkpoint_survivors.read_text())["survivors"] + gate_mass = np.load(args.imatrix)["reap__gate_norm_sum"].astype(np.float64) # [L, E_local] + e1, e2 = layer_eps(args.metrics_1bit), layer_eps(args.metrics_2bit) + keep, hot, rows = {}, {}, [] + for layer in range(len(survivors)): + ids = sorted(survivors[str(layer)]) + mass = gate_mass[layer] / gate_mass[layer].sum() + eps = (1.0, e1[layer], e2[layer]) + rung = allocate(mass, eps, args.budget) + keep[str(layer)] = [ids[i] for i in np.flatnonzero(rung >= 1)] + hot[str(layer)] = [ids[i] for i in np.flatnonzero(rung == 2)] + rows.append({"layer": layer, "dropped": int((rung == 0).sum()), "rung1": int((rung == 1).sum()), + "rung2": int((rung == 2).sum()), "eps1": round(eps[1], 4), "eps2": round(eps[2], 4), + "est_cost": float((mass * np.asarray(eps)[rung]).sum())}) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "keep.json").write_text(json.dumps({"survivors": keep})) + (args.out / "hot.json").write_text(json.dumps({"survivors": hot})) + (args.out / "alloc.json").write_text(json.dumps({"budget": args.budget, "layers": rows}, indent=1)) + print(json.dumps({"budget": args.budget, "kept_mean": float(np.mean([len(v) for v in keep.values()])), + "hot_mean": float(np.mean([len(v) for v in hot.values()])), + "est_cost_mean": float(np.mean([r["est_cost"] for r in rows]))})) + return 0 + + +def _check() -> None: + eps = (1.0, 0.3, 0.1) + # flat mass: a 1-bit floor for everyone before any upgrade (0.175 per unit vs 0.05) + assert allocate(np.full(4, 0.25), eps, 4).tolist() == [1, 1, 1, 1] + m = np.array([0.5, 0.3, 0.15, 0.05]) + assert allocate(m, eps, 3).tolist() == [1, 1, 1, 0] + # upgrading the 0.5 expert (0.10) beats keeping the 0.05 one (0.035) + assert allocate(m, eps, 4).tolist() == [2, 1, 1, 0] + assert allocate(m, eps, 5).tolist() == [2, 2, 1, 0] + + +if __name__ == "__main__": + import sys + + if sys.argv[1:] == ["--check"]: + _check() + print("ok") + sys.exit(0) + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/requirements-ref.txt b/scripts/runpod/v41_reap/requirements-ref.txt new file mode 100644 index 0000000..0ac24b8 --- /dev/null +++ b/scripts/runpod/v41_reap/requirements-ref.txt @@ -0,0 +1,9 @@ +torch>=2.10.0 +transformers +tokenizers +safetensors>=0.7.0 +numpy +sympy +Pillow +tilelang==0.1.12 +tqdm diff --git a/scripts/runpod/v41_reap/sidecar_fit.py b/scripts/runpod/v41_reap/sidecar_fit.py new file mode 100644 index 0000000..14b06f9 --- /dev/null +++ b/scripts/runpod/v41_reap/sidecar_fit.py @@ -0,0 +1,314 @@ +"""Step 3: layer-local residual sidecars for a pruned, KEEP-quantized V4.1 student, in one pass. + +The student is a survivors set (base expert ids) whose experts run from KEEP codes as bf16 GEMMs. +The teacher is the same layer with every expert the checkpoint holds, at its own FP4 weights. In +one layer-streamed pass over the student's own hidden states, each layer's MoE computes both: + + r = y_teacher(x) - y_student(x) routed experts only; the shared expert cancels + +and fits a reduced-rank linear map x~ = [x, 1] -> r by ridge regression on the fit tokens: + + W = (X~'X~ + lam * mean(diag) * I)^-1 X~'R, then the best rank-k W V V' (V: top eigvecs of W'X~'X~W) + +The rank-k sidecar is added to the student's MoE output before the next layer runs, so every +later layer is fit on inputs that already carry the earlier sidecars (sequential, like GPTQ). +Held-out records score each layer: the share of teacher output energy the residual holds, +before and after the sidecar. + +On the pod the checkpoint is the full 384-expert source; on the Mac, REAP-50 stands in as a +proxy teacher for smoke tests. Student codes come from ``pack-codes`` (only the student's +experts, keyed by base id), so the pod never needs the full code directory: + + python sidecar_fit.py pack-codes --codes ~/Models/v41-reap-run/quant/e8p-podimp \\ + --checkpoint-survivors ~/Models/v41-reap-run/stats/survivors-192.json \\ + --student ~/Models/v41-reap-run/alloc/sel/prunedmass-96.json --out ~/Models/v41-reap-run/sidecar/codes + + python sidecar_fit.py fit --hf-path HF --pack PACK --records RECORDS.json \\ + --student SURVIVORS.json --codes CODES_DIR --rank 256 --out OUT + +``RECORDS.json`` is a list of ``session#index`` names; every ``--holdout-every``-th one is held out. +Writes ``OUT/layer-LL.npz`` (U [dim+1, k], Vt [k, dim], fp32; optional X~'X~ and X~'R with +``--save-stats``) and ``OUT/metrics.jsonl``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import numpy as np + +KIT = Path(__file__).resolve().parent +sys.path.insert(0, str(KIT)) +sys.path.insert(0, str(KIT.parents[2] / "src")) + + +def local_ids(hf_path: Path, ckpt_survivors: Path | None, n_experts: int) -> dict[int, list[int]]: + """Per layer, the base expert id of each local expert slot in this checkpoint.""" + + if not (hf_path / "keep-prune.json").exists(): + return {layer: list(range(n_experts)) for layer in range(64)} + assert ckpt_survivors is not None, "a pruned checkpoint needs --checkpoint-survivors" + return {int(k): sorted(v) for k, v in json.loads(ckpt_survivors.read_text())["survivors"].items()} + + +def pack_codes(args: argparse.Namespace) -> int: + have = {int(k): sorted(v) for k, v in json.loads(args.checkpoint_survivors.read_text())["survivors"].items()} + want = {int(k): sorted(v) for k, v in json.loads(args.student.read_text())["survivors"].items()} + args.out.mkdir(parents=True, exist_ok=True) + for layer in sorted(want): + local = [have[layer].index(e) for e in want[layer]] + with np.load(args.codes / f"layer-{layer:02d}.npz") as z: + out = {"ids": np.asarray(want[layer], dtype=np.int32), + "code_bits": z["code_bits"] if "code_bits" in z.files else np.asarray(16)} + for proj in ("w1", "w2", "w3"): + out[f"{proj}_codes"] = z[f"{proj}_codes"][local] + out[f"{proj}_scales"] = z[f"{proj}_scales"][local] + np.savez(args.out / f"layer-{layer:02d}.npz", **out) + print(json.dumps({"layers": len(want), "experts": int(np.mean([len(v) for v in want.values()]))})) + return 0 + + +class LayerCodes: + """One ``pack-codes`` layer file, held as codes; ``[base id]`` decodes that expert to bf16. + + Decoding on use keeps only one expert's bf16 weights alive at a time (all 96 of a layer, twice + over with the prefetch, cost ~13 GB of host RAM and tripped the Mac's 80 GB guard).""" + + def __init__(self, path: Path, torch, grids): + self.torch = torch + with np.load(path) as z: + self.grid = grids[int(z["code_bits"])] + self.row = {e: i for i, e in enumerate(z["ids"].tolist())} + self.arrays = {k: z[k] for k in z.files if k.endswith(("_codes", "_scales"))} + + def __getitem__(self, e: int) -> dict[str, object]: + torch, i, out = self.torch, self.row[e], {} + with torch.device("cpu"): + for proj in ("w1", "w2", "w3"): + vec = self.grid[torch.as_tensor(self.arrays[f"{proj}_codes"][i].astype(np.int64))] + scales = torch.as_tensor(self.arrays[f"{proj}_scales"][i].astype(np.float32)) + vec = vec * scales.repeat_interleave(vec.shape[1] // scales.shape[-1], dim=-1)[..., None] + out[proj] = vec.flatten(-2).bfloat16() + return out + + +def reduced_rank(A, B, lam: float, rank: int): + """Ridge W = (A + reg)^-1 B, cut to the rank-k map that keeps the most fitted output energy.""" + + import torch + + n = A.shape[0] + reg = lam * A.diagonal()[:-1].mean() + W = torch.linalg.solve(A + reg * torch.eye(n, dtype=A.dtype, device=A.device), B) # [n, d] + M = W.T @ A @ W + _, V = torch.linalg.eigh((M + M.T) / 2) + V = V[:, -rank:].flip(-1) # largest first, so U[:, :j] @ Vt[:j] is the best rank-j map + return W @ V, V.T # U [n, k], Vt [k, d] + + +def fit(args: argparse.Namespace) -> int: + import torch + import torch.nn.functional as F + from transformers import AutoTokenizer + + import collect + import stream + from keep.vq.e8 import e8_1bit_grid, e8p_full_grid + + cuda = args.device == "cuda" + if not cuda: + import torch_kernel + + torch_kernel.exact_quant = True + ref = stream.import_reference(args.hf_path / "inference", kernel=torch_kernel) + else: + ref = stream.import_reference(args.hf_path / "inference") + torch.cuda.memory._set_allocator_settings("expandable_segments:True") + torch.set_default_device("cuda") + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(33377335) + model_args = ref.ModelArgs(**json.loads((args.hf_path / "inference" / "config.json").read_text())) + model_args.max_batch_size = 1 + dim, n_layers = int(model_args.dim), int(model_args.n_layers) + + sessions = collect.load_sessions(args.pack) + names = json.loads(args.records.read_text()) + records = [] + for name in names: + session, _, idx = name.partition("#") + r = sessions[session][int(idx or 0)] + records.append(dict(r, token_ids=r["token_ids"][: args.max_tokens])) + model_args.max_seq_len = max(len(r["token_ids"]) for r in records) + holdout = torch.cat([torch.full((len(r["token_ids"]),), i % args.holdout_every == args.holdout_every - 1) + for i, r in enumerate(records)]).to(args.device) + + ckpt = stream.HFCheckpoint(args.hf_path) + layout = ref.EngramLayout.from_args(model_args) + with torch.device(args.device): + hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) + engram_rows = {l: stream.EngramRows.from_checkpoint(ckpt, l, ref.fp8_block_size) for l in model_args.engram_layer_ids} + streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, args.device) + + ids = local_ids(args.hf_path, args.checkpoint_survivors, int(model_args.n_routed_experts)) + student = {int(k): v for k, v in json.loads(args.student.read_text())["survivors"].items()} + grids = {16: torch.as_tensor(e8p_full_grid().astype(np.float32), device="cpu"), + 8: torch.as_tensor(e8_1bit_grid().astype(np.float32), device="cpu")} + decoded: dict[int, dict] = {} + load_layer = streamer.load_layer + + def load_with_codes(prefix, skip=()): # prefetch thread: the next layer's student weights too + sd = load_layer(prefix, skip) + if prefix.startswith("layers."): + layer = int(prefix.split(".")[1]) + decoded[layer] = LayerCodes(args.codes / f"layer-{layer:02d}.npz", torch, grids) + return sd + + streamer.load_layer = load_with_codes + args.out.mkdir(parents=True, exist_ok=True) + metrics = (args.out / "metrics.jsonl").open("a") + + def expert(x, w, w1, w2, w3, limit): + gate, up = F.linear(x, w1).float(), F.linear(x, w3).float() + if limit > 0: + up, gate = up.clamp(-limit, limit), gate.clamp(max=limit) + return F.linear((w * (F.silu(gate) * up)).to(x.dtype), w2) + + def moe_forward(self, x, image_mask=None): + t0 = time.time() + shape, layer = x.size(), self.layer_id + x = x.view(-1, dim) + local = ids[layer] + full_bias = self.gate.bias.data.clone() + weights, indices = self.gate(x, None) # teacher: every expert in the checkpoint + y_t = torch.zeros_like(x, dtype=torch.float32) + for i in torch.unique(indices).tolist(): + idx, top = torch.where(indices == i) + y_t[idx] += self.experts[i](x[idx], weights[idx, top, None]).float() + keep = set(student[layer]) + self.gate.bias.data[torch.as_tensor([e not in keep for e in local], device=x.device)] = float("-inf") + weights, indices = self.gate(x, None) # student: masked router, KEEP-decoded experts + self.gate.bias.data.copy_(full_bias) + weights_of = decoded.pop(layer) + y_s = torch.zeros_like(x, dtype=torch.float32) + limit = self.experts[0].swiglu_limit if self.experts[0] is not None else 0.0 + for i in torch.unique(indices).tolist(): + idx, top = torch.where(indices == i) + w = {p: t.to(x.device) for p, t in weights_of[local[i]].items()} + y_s[idx] += expert(x[idx], weights[idx, top, None], w["w1"], w["w2"], w["w3"], limit).float() + + energy = {s: float(y_t[m].pow(2).sum()) for s, m in (("fit", ~holdout), ("holdout", holdout))} + r = y_t.sub_(y_s) # in place: y_t is not needed past its energy + del y_t + n = x.size(0) + A = torch.zeros(dim + 1, dim + 1, dtype=torch.float64, device=x.device) + B = torch.zeros(dim + 1, dim, dtype=torch.float64, device=x.device) + + def blocks(): # [x, 1] in row blocks, so no full fp32 copy of x is ever resident + for s in range(0, n, args.block_rows): + e = min(n, s + args.block_rows) + yield s, e, torch.cat([x[s:e].float(), torch.ones(e - s, 1, device=x.device)], dim=1) + + for s, e, xt in blocks(): + m = ~holdout[s:e] + A += (xt[m].T @ xt[m]).double() + B += (xt[m].T @ r[s:e][m]).double() + U, Vt = reduced_rank(A, B, args.ridge, args.rank) + U, Vt = U.float(), Vt.float() + before = {"fit": 0.0, "holdout": 0.0} + after = {"fit": 0.0, "holdout": 0.0} + for s, e, xt in blocks(): + corr = (xt @ U) @ Vt + for split, m in (("fit", ~holdout[s:e]), ("holdout", holdout[s:e])): + before[split] += float(r[s:e][m].pow(2).sum()) + after[split] += float((r[s:e][m] - corr[m]).pow(2).sum()) + if args.apply: + y_s[s:e] += corr + del r + row = {"layer": layer, "tokens_fit": int((~holdout).sum()), "tokens_holdout": int(holdout.sum()), + "seconds": round(time.time() - t0, 1)} + for split in ("fit", "holdout"): + row[f"{split}_res_before"] = before[split] / energy[split] + row[f"{split}_res_after"] = after[split] / energy[split] + out = {"U": U.cpu().numpy(), "Vt": Vt.cpu().numpy(), "rank": np.asarray(args.rank), "ridge": np.asarray(args.ridge)} + if args.save_stats: + out |= {"XtX": A.float().cpu().numpy(), "XtR": B.float().cpu().numpy()} + np.savez(args.out / f"layer-{layer:02d}.npz", **out) + metrics.write(json.dumps(row) + "\n") + metrics.flush() + print(json.dumps(row), flush=True) + y_s += self.shared_experts(x).float() + return y_s.type_as(x).view(shape) + + ref.MoE.forward = moe_forward + items = [stream.Item(tokens=torch.as_tensor([r["token_ids"]], dtype=torch.long, device=args.device), group="student") + for r in records] + t0 = time.time() + with torch.inference_mode(): + streamer.forward(items) + summary = {"seconds": round(time.time() - t0, 1), "tokens": int(holdout.numel()), "layers": n_layers} + if cuda: + summary["max_gpu_gb"] = round(torch.cuda.max_memory_allocated() / 2**30, 1) + print(json.dumps(summary), flush=True) + return 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + pc = sub.add_parser("pack-codes") + pc.add_argument("--codes", required=True, type=Path) + pc.add_argument("--checkpoint-survivors", required=True, type=Path) + pc.add_argument("--student", required=True, type=Path) + pc.add_argument("--out", required=True, type=Path) + f = sub.add_parser("fit") + f.add_argument("--hf-path", required=True, type=Path) + f.add_argument("--checkpoint-survivors", type=Path, help="only for a pruned checkpoint (the Mac proxy)") + f.add_argument("--pack", required=True, type=Path) + f.add_argument("--records", required=True, type=Path) + f.add_argument("--max-tokens", type=int, default=1024) + f.add_argument("--holdout-every", type=int, default=10) + f.add_argument("--student", required=True, type=Path) + f.add_argument("--codes", required=True, type=Path) + f.add_argument("--rank", type=int, default=256) + f.add_argument("--ridge", type=float, default=1e-3) + f.add_argument("--no-apply", dest="apply", action="store_false", help="fit only; do not feed sidecars forward") + f.add_argument("--save-stats", action="store_true") + f.add_argument("--block-rows", type=int, default=32768, help="rows per [x, 1] block in the fit") + f.add_argument("--device", default="cuda") + f.add_argument("--out", required=True, type=Path) + f.add_argument("--max-rss-gb", type=float, default=80.0) + args = p.parse_args(argv) + if args.cmd == "pack-codes": + return pack_codes(args) + from local_parity import rss_guard + + rss_guard(args.max_rss_gb) + return fit(args) + + +def _check() -> None: + import torch + + torch.manual_seed(0) + x = torch.randn(4000, 16, dtype=torch.float64) + xt = torch.cat([x, torch.ones(4000, 1, dtype=torch.float64)], 1) + true = torch.randn(17, 3, dtype=torch.float64) @ torch.randn(3, 16, dtype=torch.float64) # rank 3 + r = xt @ true + U, Vt = reduced_rank(xt.T @ xt, xt.T @ r, 1e-9, 3) + assert torch.allclose(U @ Vt, true, atol=1e-6), (U @ Vt - true).abs().max() + U3, Vt3 = U, Vt + U, Vt = reduced_rank(xt.T @ xt, xt.T @ r, 1e-9, 1) + assert ((xt @ U @ Vt - r).pow(2).sum() / r.pow(2).sum()) < 1 # rank 1 keeps part of it + assert torch.allclose(U3[:, :1] @ Vt3[:1], U @ Vt, atol=1e-6) # stored largest first: truncation works + + +if __name__ == "__main__": + if sys.argv[1:] == ["--check"]: + _check() + print("ok") + sys.exit(0) + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/sidecar_pod.sh b/scripts/runpod/v41_reap/sidecar_pod.sh new file mode 100755 index 0000000..4a33e0d --- /dev/null +++ b/scripts/runpod/v41_reap/sidecar_pod.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Step 3 on one H100 pod: fetch the source checkpoint, then fit the residual sidecars. +# The Mac uploads /workspace/in/ (pack, records, student survivors, student codes) meanwhile +# and touches /workspace/in/READY when it is complete. Writes /workspace/sidecar/ and DONE. +set -euo pipefail +W=/workspace +KIT=$W/keep/scripts/runpod/v41_reap +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$W/pod.log"; } + +log "fetch + pip in parallel" +bash "$KIT/pod.sh" fetch > "$W/fetch.log" 2>&1 & +fetch_pid=$! +pip install -q -r "$KIT/requirements-ref.txt" > "$W/pip.log" 2>&1 # the pinned inference/requirements.txt +wait "$fetch_pid" || { log "fetch failed: see $W/fetch.log"; exit 1; } +log "fetch verified; waiting for the Mac upload" +while [[ ! -f "$W/in/READY" ]]; do sleep 30; done +log "fit start" +python3 "$KIT/sidecar_fit.py" fit --hf-path "$W/v41-hf" --pack "$W/in/v41-pack.jsonl" \ + --records "$W/in/records-pod.json" --student "$W/in/prunedmass-96.json" --codes "$W/in/codes" \ + --rank 256 --save-stats --device cuda --max-rss-gb 400 --out "$W/sidecar" > "$W/fit.log" 2>&1 \ + || { log "fit failed: see $W/fit.log"; exit 1; } +log "fit done: $(tail -1 "$W/fit.log")" +touch "$W/DONE" diff --git a/scripts/runpod/v41_reap/torch_kernel.py b/scripts/runpod/v41_reap/torch_kernel.py index 23cec13..47f7610 100644 --- a/scripts/runpod/v41_reap/torch_kernel.py +++ b/scripts/runpod/v41_reap/torch_kernel.py @@ -1,11 +1,15 @@ """Plain-torch stand-ins for the reference ``inference/kernel.py`` (tilelang, GPU only). -Used by the CPU tests to run DeepSeek's real ``model.py`` at toy size: install it -as ``sys.modules["kernel"]`` before importing ``model``. It covers the bf16 path -only. The quantizers are no-ops (they fake-quantize in place on the GPU) and the -quantized GEMMs refuse to run, so a test model must use ``dtype="bf16"`` and -``expert_dtype=None``. ``sparse_attn`` and ``hc_split_sinkhorn`` are exact ports -of the kernels' math. +Install as ``sys.modules["kernel"]`` before importing DeepSeek's ``model.py``. Two uses: + +- CPU tests at toy size, with bf16 weights. The in-place quantizers stay no-ops there, because toy + head sizes are not multiples of the quantization blocks. +- The real FP8/FP4 checkpoint on a machine without the GPU kernels (the M4 Max). Set + ``exact_quant = True``. Every function is then a port of the kernel's math: the same block + scales, power-of-2 rounding and clamps. The GEMMs dequantize their operands and multiply in + fp32, so only the summation order differs from the kernels. + +``sparse_attn`` and ``hc_split_sinkhorn`` are always exact ports. """ from __future__ import annotations @@ -13,24 +17,98 @@ import torch -def act_quant(x, block_size=128, scale_fmt=None, scale_dtype=torch.float32, inplace=False): - if inplace: - return x - raise NotImplementedError("fp8 activations: CPU tests use bf16 weights") +exact_quant = False # True: in-place quantizers fake-quantize like the GPU kernels (needed for real weights) + +FP8_MAX = 448.0 +FP4_MAX = 6.0 +# e2m1 magnitudes by 3-bit code; bit 3 of the nibble is the sign +E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) + +def _pow2_ceil(v: torch.Tensor) -> torch.Tensor: + """2 ** ceil(log2(v)) for positive fp32 v, by the kernel's bit trick (fast_round_scale).""" + bits = v.float().contiguous().view(torch.int32) + exp = ((bits >> 23) & 0xFF) - 127 + ((bits & 0x7FFFFF) != 0).to(torch.int32) + return ((exp + 127) << 23).view(torch.float32) -def fp4_act_quant(x, block_size=32, inplace=False, scale_dtype=None): + +def _blocks(x: torch.Tensor, block_size: int) -> torch.Tensor: + assert x.size(-1) % block_size == 0, (x.shape, block_size) + return x.float().unflatten(-1, (-1, block_size)) # [..., n_blocks, block] + + +def act_quant(x, block_size=128, scale_fmt=None, scale_dtype=torch.float32, inplace=False): + """Block-wise FP8 e4m3 quantization of activations (``act_quant_kernel``).""" + if inplace and not exact_quant: + return x + xb = _blocks(x, block_size) + amax = xb.abs().amax(dim=-1).clamp_min(1e-4) + s = amax * torch.tensor(1 / FP8_MAX, dtype=torch.float32) + if scale_fmt is not None: + s = _pow2_ceil(s) + y = (xb / s[..., None]).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) if inplace: + x.copy_((y.float() * s[..., None]).flatten(-2).to(x.dtype)) return x - raise NotImplementedError("fp4 activations: CPU tests use bf16 weights") + return y.flatten(-2), s.to(scale_dtype) -def fp8_gemm(*args, **kwargs): - raise NotImplementedError("fp8 GEMM: CPU tests use bf16 weights") +def _e2m1_round(v: torch.Tensor) -> torch.Tensor: + """Nearest e2m1 value, ties to the even code (``T.Cast(FP4, ...)``); v already in [-6, 6].""" + mag = v.abs() + grid = E2M1.to(v.device) + dist = (mag[..., None] - grid).abs() + best = dist.amin(dim=-1, keepdim=True) + codes = torch.arange(8, device=v.device) + # among equally near codes take the even one (its mantissa bit is 0) + pick = torch.where(dist == best, codes % 2, 99).argmin(dim=-1) # ties are adjacent codes: one even + return torch.copysign(grid[pick], v) -def fp4_gemm(*args, **kwargs): - raise NotImplementedError("fp4 GEMM: CPU tests use bf16 weights") +def fp4_act_quant(x, block_size=32, inplace=False, scale_dtype=torch.float8_e8m0fnu): + """Block-wise FP4 fake quantization (``fp4_quant_kernel``); only the in-place form is used.""" + assert inplace, "model.py only calls fp4_act_quant in place" + if not exact_quant: + return x + xb = _blocks(x, block_size) + amax = xb.abs().amax(dim=-1) + if scale_dtype == torch.float8_e4m3fn: + s = (amax.clamp_min(6 * 2**-9) / FP4_MAX).to(torch.float8_e4m3fn).float() + else: + s = _pow2_ceil(amax.clamp_min(6 * 2**-126) * torch.tensor(1 / FP4_MAX, dtype=torch.float32)) + y = _e2m1_round((xb / s[..., None]).clamp(-FP4_MAX, FP4_MAX)) + x.copy_((y * s[..., None]).flatten(-2).to(x.dtype)) + return x + + +def _dequant_rows(values: torch.Tensor, scale: torch.Tensor, block: int) -> torch.Tensor: + """values [R, K] fp32 times a per-(row, K-block) scale [R, K/block].""" + return (values.unflatten(-1, (-1, block)) * scale.float()[..., None]).flatten(-2) + + +def decode_e2m1(b: torch.Tensor) -> torch.Tensor: + """float4_e2m1fn_x2 [N, K/2] to fp32 [N, K]; the low nibble is the first element.""" + codes = b.contiguous().view(torch.uint8) + nib = torch.stack((codes & 0x0F, codes >> 4), dim=-1).flatten(-2).long() + mag = E2M1.to(b.device)[nib & 0x7] + return torch.where((nib & 0x8) != 0, -mag, mag) + + +def fp8_gemm(a, a_s, b, b_s, scale_dtype=torch.float32, block_size=128): + """C[M,N] = A[M,K] @ B[N,K]^T; A scaled per (row, K block), B per (N block, K block).""" + K = a.size(-1) + A = _dequant_rows(a.reshape(-1, K).float(), a_s.reshape(-1, K // block_size), block_size) + sb = b_s.float().repeat_interleave(block_size, dim=0)[: b.size(0)] + B = _dequant_rows(b.float(), sb, block_size) + return (A @ B.T).to(torch.get_default_dtype()).view(*a.shape[:-1], b.size(0)) + + +def fp4_gemm(a, a_s, b, b_s, scale_dtype=torch.float32, act_block_size=128): + """C[M,N] = A_fp8[M,K] @ B_fp4[N,K]^T; B scaled per 32 along K (e8m0).""" + K = a.size(-1) + A = _dequant_rows(a.reshape(-1, K).float(), a_s.reshape(-1, K // act_block_size), act_block_size) + B = _dequant_rows(decode_e2m1(b), b_s, 32) + return (A @ B.T).to(torch.get_default_dtype()).view(*a.shape[:-1], b.size(0)) def sparse_attn(q, kv, attn_sink, topk_idxs, softmax_scale): diff --git a/tests/test_runpod_v41_routed_precision.py b/tests/test_runpod_v41_routed_precision.py new file mode 100644 index 0000000..6872c6d --- /dev/null +++ b/tests/test_runpod_v41_routed_precision.py @@ -0,0 +1,64 @@ +"""Routed-precision pieces of the V4.1 REAP recovery kit: the residual cache, the reduced-rank +sidecar fit, the static precision allocator and the nested-code decoder.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "scripts" / "runpod" / "v41_reap")) +sys.path.insert(0, str(REPO / "src")) + +import local_eval # noqa: E402 +import precision_alloc # noqa: E402 + + +def test_residual_cache_hits_only_after_async_admission(): + ind = np.array([[0, 1], [0, 1], [2, 0]]) + wts = np.array([[0.5, 0.1], [0.5, 0.1], [0.5, 0.5]]) + counts = np.zeros((3, 5)) + hi = local_eval.residual_cache(ind, wts, k=1, tau=0.3, bounds=[3], warm=None, counts=counts) + # token 0: cold; expert 0 admitted. token 1: expert 0 resident, expert 1 never admitted (gate < tau). + # token 2: expert 2 not yet resident, expert 0 still is. Then 2 is admitted (evicting 0, k=1) and 0 + # is admitted again (evicting 2): one slot of cache thrashes, and each admission is a fetch. + assert hi.tolist() == [[False, False], [True, False], [False, True]] + assert counts[:, 4].tolist() == [2, 0, 1] + + +def test_residual_cache_resets_per_record_and_warm_start(): + ind = np.array([[0], [0], [0]]) + wts = np.ones((3, 1)) + counts = np.zeros((1, 5)) + hi = local_eval.residual_cache(ind, wts, k=1, tau=0.3, bounds=[2, 1], warm=None, counts=counts) + assert hi[:, 0].tolist() == [False, True, False] # the second record starts cold again + assert counts[0, 4] == 2 + hi = local_eval.residual_cache(ind, wts, k=1, tau=0.3, bounds=[3], warm=np.array([0]), counts=np.zeros((1, 5))) + assert hi.all() + + +def test_precision_allocator(): + precision_alloc._check() + + +def test_sidecar_reduced_rank(): + pytest.importorskip("torch") + import sidecar_fit + + sidecar_fit._check() + + +def test_nested_decoder_matches_local_eval(): + import keep_quant + from keep.vq.e8 import e8_1bit_grid + + rng = np.random.default_rng(0) + z = {"code_bits": np.asarray(8), "w1_codes": rng.integers(0, 256, (2, 4, 64), dtype=np.uint16), + "w1_scales": rng.random((2, 4, 1)).astype(np.float16)} + got = keep_quant.decode_codes(z, "w1", 1) + grid = e8_1bit_grid().astype(np.float32) + want = (grid[z["w1_codes"][1].astype(np.int64)] * z["w1_scales"][1].astype(np.float32)[..., None]).reshape(4, -1) + assert np.allclose(got, want) diff --git a/tests/test_runpod_v41_torch_kernel.py b/tests/test_runpod_v41_torch_kernel.py new file mode 100644 index 0000000..c16375b --- /dev/null +++ b/tests/test_runpod_v41_torch_kernel.py @@ -0,0 +1,96 @@ +"""The FP8/FP4 stand-ins in torch_kernel.py against block-loop references written like the kernels.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "scripts" / "runpod" / "v41_reap")) +sys.path.insert(0, str(REPO / "src")) + +import torch_kernel as k # noqa: E402 + + +@pytest.fixture(autouse=True) +def _keep_default_dtype(): + """The GEMMs return the default dtype; set fp32 here and leave other tests' default alone.""" + before = torch.get_default_dtype() + torch.set_default_dtype(torch.float32) + yield + torch.set_default_dtype(before) + + +def _e8m0(shape, lo=-6, hi=2): + return (2.0 ** torch.randint(lo, hi, shape).float()).to(torch.float8_e8m0fnu) + + +def test_decode_e2m1_matches_numpy_decoder(): + from keep.convert.fp4_expert import decode_fp4_e2m1 + + codes = torch.arange(256, dtype=torch.uint8).view(16, 16) + want = decode_fp4_e2m1(codes.numpy().view(np.int8)) + got = k.decode_e2m1(codes.view(torch.float4_e2m1fn_x2)) + np.testing.assert_array_equal(got.numpy(), np.asarray(want, dtype=np.float32)) + + +def test_act_quant_scales_are_pow2_and_inplace_is_quant_dequant(): + torch.manual_seed(0) + x = torch.randn(5, 96, dtype=torch.bfloat16) * 7 + y, s = k.act_quant(x, 32, "ue8m0", torch.float8_e8m0fnu) + sf = s.float() + assert torch.equal(sf, 2.0 ** torch.log2(sf).round()) # powers of two + amax = x.float().unflatten(-1, (-1, 32)).abs().amax(-1) + assert (amax / sf <= 448).all() and (amax / sf > 224).all() # the smallest pow2 scale that fits + k.exact_quant = True + try: + z = x.clone() + k.act_quant(z, 32, "ue8m0", torch.float8_e8m0fnu, inplace=True) + finally: + k.exact_quant = False + deq = (y.float().unflatten(-1, (-1, 32)) * sf[..., None]).flatten(-2).to(torch.bfloat16) + assert torch.equal(z, deq) + + +def _block_loop(A, sa, B, sb_of, K, kb): + """sum over K blocks of (A_k @ B_k^T) * sa[:, k] * sb[:, k], as the tilelang kernels accumulate.""" + out = torch.zeros(A.shape[0], B.shape[0]) + for i in range(K // kb): + part = A[:, i * kb : (i + 1) * kb] @ B[:, i * kb : (i + 1) * kb].T + out += part * sa[:, i : i + 1] * sb_of(i)[None, :] + return out + + +def test_fp8_gemm_matches_block_loop(): + torch.manual_seed(1) + M, N, K, bs = 7, 70, 96, 32 + a = (torch.randn(M, K) * 50).clamp(-448, 448).to(torch.float8_e4m3fn) + b = (torch.randn(N, K) * 50).clamp(-448, 448).to(torch.float8_e4m3fn) + a_s, b_s = _e8m0((M, K // bs)), _e8m0(((N + bs - 1) // bs, K // bs)) + got = k.fp8_gemm(a, a_s, b, b_s, torch.float8_e8m0fnu, bs) + rows = torch.arange(N) // bs + want = _block_loop(a.float(), a_s.float(), b.float(), lambda i: b_s.float()[rows, i], K, bs) + torch.testing.assert_close(got, want, rtol=1e-5, atol=1e-3) + + +def test_fp4_gemm_matches_block_loop(): + torch.manual_seed(2) + M, N, K, act_bs = 6, 40, 128, 32 + a = (torch.randn(M, K) * 50).clamp(-448, 448).to(torch.float8_e4m3fn) + b = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8).view(torch.float4_e2m1fn_x2) + a_s, b_s = _e8m0((M, K // act_bs)), _e8m0((N, K // 32)) + got = k.fp4_gemm(a, a_s, b, b_s, torch.float8_e8m0fnu, act_bs) + B = k.decode_e2m1(b) + want = _block_loop(a.float(), a_s.float().repeat_interleave(act_bs // 32, 1), B, lambda i: b_s.float()[:, i], K, 32) + torch.testing.assert_close(got, want, rtol=1e-5, atol=1e-3) + + +def test_e2m1_round_ties_to_even(): + v = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, -0.25, -2.5, 6.0, 0.3]) + want = [0.0, 1.0, 1.0, 2.0, 2.0, 4.0, 4.0, -0.0, -2.0, 6.0, 0.5] + assert k._e2m1_round(v).tolist() == want From 5252788daaee722c27031148aa7ab76fc812b16a Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:38:15 -0800 Subject: [PATCH 27/74] v41 routed precision: per-token tier 0 (--tau-drop), cost report, paper-track plan in the log --- .../2026-11-08-v41-reap-recovery-log.md | 61 +++++++++++++++++ scripts/runpod/v41_reap/local_eval.py | 22 ++++-- scripts/runpod/v41_reap/routed_report.py | 67 +++++++++++++++++++ 3 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 scripts/runpod/v41_reap/routed_report.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 8e6f276..dacfa13 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -516,3 +516,64 @@ Learnings: helps only marginally (0.025 at 192 units, nothing at 288). A low rung that pays needs to be better than this 1-bit E8 fit (errors 0.31/0.25), or dynamic, or both. - Step 2 is done; step 3 (residual sidecars) is next. + +## Precision as a routed dimension (the paper track) + +Working title: *Precision as a Routed Dimension: unifying pruning, quantization and offloading in +MoE inference*. Jack, 2026-11-09: build, run, test, commit; this is the headline to reach. + +### Claim to test + +Static compression picks one precision per expert (or drops it) before inference. The router +already decides, per token, how much each expert matters (its gate weight). Make precision a +routed decision too: every (token, expert) slot runs at one of three tiers chosen at inference +time, from codes nested so that each tier is a prefix of the next: + +| Tier | Weights | Lives in | Cost per use | +|---|---|---|---| +| 0 (drop) | none | nowhere | 0 | +| 1 (base) | 1-bit E8 code | unified memory, always resident | 4.6 MB read | +| 2 (base + residual) | 1-bit base + 1-bit E8 residual of it | residual in an LRU cache, misses on SSD | 9.2 MB read | + +Pruning is tier 0 (per expert or per token), quantization is the tier-1/tier-2 choice, and +offloading is where the tier-2 residual lives. One policy (gate thresholds `tau_drop`, `tau`, and +the cache size K) sets all three, and three costs fall out of it: resident memory (bases + K +residuals per layer), SSD traffic (residual admissions per token), and unified-memory bytes read +per token (the decode speed bound on Apple Silicon, 546 GB/s on the M4 Max). + +The cache never stalls decode: a slot gets tier 2 only if its residual is already resident; a slot +with gate >= tau admits its residual by an async SSD read that serves later tokens. This is the +HOBBIT-style "use a cheaper copy on a miss" (arXiv 2411.01433), with the cheaper copy being the +prefix of the expensive one, so a hit costs only the residual's bytes and memory holds no +duplicate. + +### Evidence so far + +- Nested codes are almost free: on layer 5 (2 experts), base + residual scores 0.103 (gate/up) and + 0.083 (down) weighted relative MSE, against 0.090 / 0.074 for a flat 2-bit E8P fit (all-layer + means). The base alone is 0.31 / 0.25. +- Static per-expert allocation (step 2c) beat REAP-75 + KEEP by 0.20 KL at equal memory, but the + ablation (2c, 2d) traced almost all of it to selection; a static 1-bit tier added 0.025. The + routed version has to beat that static frontier to carry the paper. + +### Tooling + +- `keep_quant.py --residual-of DIR`: the nested residual tier (`quant/e8-nest-r1`, all 192 experts + of REAP-50, running). +- `local_eval.py --dynamic BASE+RESIDUAL --tau NAME=t --tau-drop NAME=t --cache-k NAME=K + --cache-warm NPZ`: per-slot tiers in the forward, with the LRU simulated in token order per + record (`residual_cache`, unit-tested). Writes `tier-counts.npy` per variant: per layer and + expert, slots and gate mass at each tier, and admissions. + +### The experiments that decide it + +1. Endpoints: all tier 2 (tau 0), all tier 1 (tau 1e9). Must land near flat 2-bit (0.747) and + flat 1-bit (1.358). +2. Routed tiers, everything resident (no SSD): tau sweep. Bytes read per token against KL, + compared with static allocations at equal bytes. +3. Routed tiers with offload: K in {48, 96} residuals per layer, tau in {0.2, 0.3}, warm-started + with the pruned-model hot set. Resident units = 192 + K. The bar: beat prec-288 (0.809 at 288 + units, nothing streamed) at 288 units with an SSD rate the M4 Max sustains (about 5 GB/s + sequential; well under that at 30 tok/s). +4. Per-token tier 0 (tau_drop): does skipping low-gate slots cost less KL than static pruning + saves? diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 208f037..fa34a9a 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -175,6 +175,8 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--dynamic", help="BASE_DIR+RESIDUAL_DIR: per-token tiers, base alone or base + nested residual") p.add_argument("--tau", action="append", default=[], metavar="NAME=VALUE", help="with --dynamic, per variant: a slot runs base + residual when its gate weight >= VALUE") + p.add_argument("--tau-drop", action="append", default=[], metavar="NAME=VALUE", + help="with --dynamic, per variant: a slot with gate weight < VALUE is skipped (rung 0 per token)") p.add_argument("--cache-k", action="append", default=[], metavar="NAME=K", help="with --dynamic: hi tier only from an LRU of K resident residuals per layer, admitted at gate >= tau") p.add_argument("--cache-warm", type=Path, help="imatrix npz (reap__gate_norm_sum): warm-start each cache with the top-K") @@ -239,6 +241,7 @@ def main(argv: list[str] | None = None) -> int: base_dir, _, res_dir = args.dynamic.partition("+") base_dir, res_dir = Path(base_dir).expanduser(), Path(res_dir).expanduser() taus = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau)} + tau_drop = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau_drop)} assert set(taus) == set(variants), "--tau needs one NAME=VALUE per --variant" np_grids = {16: e8p_full_grid().astype(np.float32), 8: e8_1bit_grid().astype(np.float32)} cache: dict = {"layer": None} @@ -247,8 +250,8 @@ def main(argv: list[str] | None = None) -> int: warm = None if args.cache_warm: # warm start: each layer's top-K experts by routed mass (the static hot set) warm = np.argsort(-np.load(args.cache_warm)["reap__gate_norm_sum"], axis=1, kind="stable") - for v in variants: # [layer, expert, (lo slots, hi slots, lo gate mass, hi gate mass, admissions)] - dyn_counts[v] = np.zeros((n_layers_, n_local, 5)) + for v in variants: # [layer, expert, (lo slots, hi slots, lo mass, hi mass, admissions, dropped slots, dropped mass)] + dyn_counts[v] = np.zeros((n_layers_, n_local, 7)) def simulate_cache(layer, ind, wts, k, tau, counts): return residual_cache(ind, wts, k, tau, state["bounds"], None if warm is None else warm[layer, :k], counts) @@ -295,8 +298,12 @@ def dynamic_forward(self, x, image_mask=None): idx, top = torch.where(indices == i) w = weights[idx, top] hi_mask = w >= tau if hi_all is None else hi_all[idx, top] + drop = w < tau_drop.get(state["variant"], 0.0) # rung 0 per token: the slot is skipped + counts[i, 5] += int(drop.sum()) + counts[i, 6] += float(w[drop].sum()) + hi_mask = hi_mask & ~drop lo_w, hi_w = tier_weights(layer, i) - for mask, tier, col in ((~hi_mask, lo_w, 0), (hi_mask, hi_w, 1)): + for mask, tier, col in ((~hi_mask & ~drop, lo_w, 0), (hi_mask, hi_w, 1)): if mask.any(): y[idx[mask]] += run_expert(x[idx[mask]], w[mask, None], tier, limit).float() counts[i, col] += int(mask.sum()) @@ -435,9 +442,12 @@ def before_moe(layer: int, block, key) -> None: np.save(args.out / v / "tier-counts.npy", c) tot = c.sum(axis=(0, 1)) summary["variants"][v]["_tiers"] = {"tau": taus[v], "cache_k": cache_k.get(v), - "hi_slot_share": float(tot[1] / tot[:2].sum()), - "hi_mass_share": float(tot[3] / tot[2:4].sum()), - "admissions_per_token_layer": float(tot[4] / (tot[:2].sum() / int(model_args.n_activated_experts)) / int(model_args.n_layers))} + "tau_drop": tau_drop.get(v, 0.0), + "hi_slot_share": float(tot[1] / (tot[:2].sum() + tot[5])), + "drop_slot_share": float(tot[5] / (tot[:2].sum() + tot[5])), + "hi_mass_share": float(tot[3] / (tot[2:4].sum() + tot[6])), + "drop_mass_share": float(tot[6] / (tot[2:4].sum() + tot[6])), + "admissions_per_token_layer": float(tot[4] / ((tot[:2].sum() + tot[5]) / int(model_args.n_activated_experts)) / int(model_args.n_layers))} print(v, json.dumps(summary["variants"][v]["_tiers"]), flush=True) summary["peak_rss_gb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 2**30, 2) (args.out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") diff --git a/scripts/runpod/v41_reap/routed_report.py b/scripts/runpod/v41_reap/routed_report.py new file mode 100644 index 0000000..6b5266c --- /dev/null +++ b/scripts/runpod/v41_reap/routed_report.py @@ -0,0 +1,67 @@ +"""Costs and quality of routed-precision variants (``local_eval.py --dynamic``), per variant: + +- KL / top-1 / NLL change: domain means from ``summary.json``. +- resident expert memory: every base (tier 1) plus K residuals per layer when cached, otherwise + every residual too (nothing streamed), in units per layer (1 unit = one 1-bit expert) and GB. +- SSD traffic: residual admissions per token, in MB/token and GB/s at ``--tok-s``. +- unified-memory bytes read per token by routed experts (tier 1 = base, tier 2 = base + residual), + the decode bound at ``--bandwidth-gbs``. + + python routed_report.py --eval ~/Models/v41-reap-run/evals/step4-routed [--eval ...] +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np + +DIM, INTER = 5120, 2304 +# 1-bit E8 code + fp16 group scales: gate/up group 512 (1.031 bpw), down group 256 (1.0625 bpw) +UNIT_BYTES = (2 * DIM * INTER * (1 + 16 / 512) + DIM * INTER * (1 + 16 / 256)) / 8 # ~4.6 MB + + +def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: + summary = json.loads((eval_dir / "summary.json").read_text()) + out = [] + for name, per in summary["variants"].items(): + tiers = per.get("_tiers") + row = {"variant": f"{eval_dir.name}:{name}", **{k: round(v, 4) for k, v in per["_domain_mean"].items()}} + if tiers is None: + out.append(row) + continue + c = np.load(eval_dir / name / "tier-counts.npy") # [L, E, 7] + n_layers, n_exp = c.shape[:2] + used = c[:, :, :2].sum() + c[:, :, 5].sum() # every routed slot + tokens = used / 6 / n_layers + k = tiers.get("cache_k") + units = n_exp + (k if k is not None else n_exp) + read = (c[:, :, 0].sum() * UNIT_BYTES + c[:, :, 1].sum() * 2 * UNIT_BYTES) / tokens + ssd = c[:, :, 4].sum() * UNIT_BYTES / tokens if k is not None else 0.0 + row |= { + "tau": tiers["tau"], "tau_drop": tiers.get("tau_drop", 0.0), "cache_k": k, + "hi_slots": round(tiers["hi_slot_share"], 3), "drop_slots": round(tiers.get("drop_slot_share", 0.0), 3), + "units_per_layer": units, "resident_gb": round(units * n_layers * UNIT_BYTES / 1e9, 1), + "ssd_mb_per_tok": round(ssd / 1e6, 1), "ssd_gbs": round(ssd * tok_s / 1e9, 2), + "read_gb_per_tok": round(read / 1e9, 2), "tok_s_bound": round(bandwidth_gbs * 1e9 / read, 1), + } + out.append(row) + return out + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--eval", action="append", required=True, type=Path) + p.add_argument("--tok-s", type=float, default=30.0) + p.add_argument("--bandwidth-gbs", type=float, default=546.0) + args = p.parse_args(argv) + for d in args.eval: + for row in rows(d.expanduser(), args.tok_s, args.bandwidth_gbs): + print(json.dumps(row)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3834329167d93185827e84b85b875f29e04ca8a2 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:54:42 -0800 Subject: [PATCH 28/74] docs: routing regimes by depth, headline progress rubric --- .../2026-11-08-v41-reap-recovery-log.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index dacfa13..72bfb77 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -577,3 +577,31 @@ duplicate. sequential; well under that at 30 tok/s). 4. Per-token tier 0 (tau_drop): does skipping low-gate slots cost less KL than static pruning saves? + +### Observation: two routing regimes by depth + +Share of REAP-50's routed mass (pruned-model capture) held by each layer's top 96 experts: +layers 0–19 keep 0.72–0.83, layers 20–39 keep 0.94–0.98 (layer 32 0.86 is the one dip). Early +layers route diffusely, late layers concentrate. The step-3 sidecar fit sees the same split: in +layers 4–6 the 96-expert 2-bit student misses 25–31% of the teacher MoE's output energy on held-out +tokens, and a rank-256 linear sidecar removes only 9–15% of it, while in layers 0–3 the gap is +3–11% and the sidecar removes 36–76%. A fixed per-layer budget cannot follow this; per-token tiers +can, since a diffuse layer simply routes more of its slots to the base tier. + +### Headline progress rubric + +Every major status update reports progress to the headline with this fixed rubric (weights sum +to 100; partial credit only for measured progress). + +| # | Item | Weight | Score | State (2026-11-09) | +|---|---|---:|---:|---| +| 1 | Nested codes, all layers, validated | 10 | 5 | fit 6/40 layers; +12% error vs flat 2-bit | +| 2 | Per-token routed tiers, implemented and tested | 10 | 10 | `--dynamic/--tau/--tau-drop`, smoke-tested | +| 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | +| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 0 | sweep chained after the nested fit | +| 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | code exists, unrun | +| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 0 | | +| 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | +| 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | +| 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | +| | **Total** | 100 | **26** | | From 69679844a8f9eaae9c5a4426f74888aea4ebec8d Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:12:15 -0800 Subject: [PATCH 29/74] docs: routed-precision plan to 100% with prior art; keep_quant stores 1-bit codes as uint8 --- .../2026-11-08-v41-reap-recovery-log.md | 16 ++ .../2026-11-09-routed-precision-plan.md | 235 ++++++++++++++++++ scripts/runpod/v41_reap/keep_quant.py | 5 +- 3 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 72bfb77..443e8eb 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -605,3 +605,19 @@ to 100; partial credit only for measured progress). | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | | | **Total** | 100 | **26** | | + +### 2026-11-09: prior art, plan, restart + +- A novelty search found MatMoE (arXiv 2611.01704, 1 Nov 2026: nested codes with per-expert + bit-width chosen from routing scores at run time), DynaExq (2511.15015: hot set at high precision + with async promotion), DyMoE (2603.19172), HOBBIT and MC-MoE. Per-token precision with nested + codes is no longer new by itself. What is left to claim: the nested residual as the offloaded + unit, tier 0 in the same policy, and the regime at 2 bits and below. Details and the full plan to + 100%: [2026-11-09-routed-precision-plan.md](2026-11-09-routed-precision-plan.md). +- The session restart killed both fits (sidecar at layer 9, nested at layer 8). The nested fit + resumed; the sidecar fit is sequential and restarted from layer 0. Jobs now run detached from + `~/Models/v41-reap-run/jobs/` so a session exit cannot stop them. +- 1-bit codes are stored as uint8 from now on (`keep_quant.py`); the existing 48 layer files were + converted in place and verified value-identical, freeing 38 GB. +- The first routed sweep swaps the τ = 0.41 variant for `s96`, a static hot set of 96 residuals with + no admissions: the same codes and memory as the routed cache, so the comparison isolates routing. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md new file mode 100644 index 0000000..00f8a33 --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -0,0 +1,235 @@ +# Routed precision: implementation plan to 100% + +Written 2026-11-09. Target headline: *Precision as a Routed Dimension: unifying pruning, +quantization and offloading in MoE inference*. Progress is scored with the fixed rubric in the +[recovery log](2026-11-08-v41-reap-recovery-log.md) ("Headline progress rubric"); this plan says +what has to be built, run and measured for each rubric item, in what order, and what counts as +done. Score today: **26 / 100**. + +## 1. Prior art found today, and what it leaves us + +A search on 2026-11-09 found work that already covers parts of the idea. This changes what the +paper can claim, so it comes first. + +| Paper | What it does | Overlap with us | +|---|---|---| +| MatMoE (arXiv 2611.01704, 1 Nov 2026, ICCAD 2026) | Nested ("Matryoshka") integer codes per expert; picks each expert's bit-width at run time from its routing score; llama.cpp engines on CPU and GPU | Per-token precision from routing scores with nested codes. This is the phrase "precision as a routed dimension". All weights stay resident; no offload, no pruning. | +| DynaExq (arXiv 2511.15015) | Hot experts resident at high precision, low-precision fallback for the rest, promotions and demotions applied asynchronously so the forward pass never waits | Our no-stall cache with async admission. Separate copies per precision, not nested. GPU memory, not SSD. | +| DyMoE (arXiv 2603.19172) | Run-time mixed precision by expert importance, depth-adaptive schedule, look-ahead prefetch | Depth-dependent importance (our "two routing regimes" observation), prefetch. | +| HOBBIT (arXiv 2411.01433) | On a cache miss, run a low-precision copy of a less critical expert instead of waiting for the load | Low tier as the miss path. Separate copies. | +| MC-MoE (arXiv 2410.06270) | Static mixed-precision per expert plus online per-token expert pruning | Per-token tier 0, but with static precision. | +| PMQ, GEMQ (ICML 2026), EdgeMoE | Static per-expert bit allocation | Our step 2c allocator. | + +Only abstracts were read for all but MatMoE (introduction and method overview). Step W0 below reads +them in full. + +What none of them appears to do, and what the paper therefore has to stand on: + +1. **The offloaded unit is the nested residual.** Tiers span unified memory and SSD inside one + code: the base is always resident, and a promotion reads only the residual (4.6 MB), never a + second full copy. Separate-copy systems (DynaExq, HOBBIT) pay 2 extra units of memory per hot + expert and 9.2 MB per promotion; at equal memory a nested design holds twice as many hot + experts. This is directly measurable in our harness. +2. **Tier 0 in the same policy.** Expert pruning (static tier 0), per-token slot dropping, precision + and residency all come from one rule with one trade-off parameter. +3. **The sub-2-bit lattice regime** (1 and 2 bits per weight, E8 codes), on a 384-expert, top-6 + frontier model, on a 128 GB laptop. + +The headline can stay, because "unifying pruning, quantization and offloading" is the surviving +claim. The first half of it is no longer new on its own, and the paper must cite and beat +MatMoE-style and DynaExq-style policies rather than only static baselines. If W0 finds a paper that +already offloads nested residuals under per-token tiers, the paper becomes an empirical study of +that design at 2 bits and below, and Jack decides whether to continue. + +## 2. Where things stand + +Done: per-token tiers in `local_eval.py` (`--dynamic`, `--tau`, `--tau-drop`), the residual cache +simulation (`residual_cache`, `--cache-k`, `--cache-warm`), the cost report (`routed_report.py`), +unit tests, and the static frontier to beat (192 units 1.014, 288 units 0.809, 384 units 0.747 KL). + +Running on the Mac (detached, scripts in `~/Models/v41-reap-run/jobs/`): the nested residual fit +(`quant/e8-nest-r1`, 8 of 40 layers), the step-3 sidecar fit (restarted from layer 0), and a chain +that runs the sidecar eval and then the first routed sweep (8 variants) when the fits finish. + +One unit = one expert at 1 bit per weight = 4.6 MB per layer. All Mac results are inside REAP-50 +(192 experts per layer), scored by KL against the full model's logits on 22 held-out sessions. + +## 3. Workstreams + +Each workstream lists the deliverable, how it is built, the acceptance test, and the fallback. +Times are estimates; the Mac runs one heavy CPU job and one GPU fit at a time, under the 80 GB cap. + +### W0. Related work, read in full (gates the claims; no rubric points) + +- Read MatMoE, DynaExq, DyMoE, HOBBIT, MC-MoE and GEMQ end to end. Write + `2026-11-09-routed-precision-related-work.md`: one row per paper with nested or not, per-token or + not, what is offloaded, pruning, bit range, models, hardware. +- Done when the three claims in section 1 are each confirmed or struck. Half a day, in parallel + with the fits. + +### W1. Nested codes validated (rubric item 1: 5 → 10) + +- Finish `quant/e8-nest-r1` (32 layers left, about 6.6 minutes each). +- Accept when (a) every layer's base + residual error is within 1.15× of the flat 2-bit fit, and + (b) the sweep's endpoints land where they should: all slots at tier 2 scores KL ≤ 0.80 (flat + 2-bit is 0.747), all at tier 1 reproduces flat 1-bit (1.358). +- If (b) fails: one alternating round (refit the base against `W − residual`, then the residual + again), about 7 h of GPU per round. +- Stretch, for the frontier figure: a third rung (`--residual-of` taking two directories), about + 3 bits per weight, 4.4 h and 34 GB of disk. + +### W4. Core result (rubric item 4: 0 → 25) + +The claim: at equal resident memory, routed tiers with residual offload give lower KL than any +static assignment and than a separate-copy cache, at an SSD read rate the Mac sustains. + +Budgets: B = 240, 288 and 336 units per layer (44, 53 and 62 GB of routed experts). A configuration +keeps N expert bases resident and K residual slots, N + K = B. Policies, all in the same harness on +the same codes: + +| Policy | What it is | How it runs | +|---|---|---| +| S, static-nested | hot set = top K by pruned-model routed mass, never changes | `--cache-k K --tau 1e9` with `--cache-warm` | +| R, routed-nested (ours) | LRU of K residuals, async admission at gate ≥ τ | `--cache-k K --tau τ` | +| D, separate-copy cache (DynaExq / HOBBIT style) | high tier is a second flat 2-bit copy, so a hot expert costs 2 extra units and a promotion reads 9.2 MB; K = (B − N) / 2 | new: `--dynamic BASE\|HI` | +| M, all-resident routed (MatMoE style) | every residual resident, tier by routing score | `--tau τ`, no cache; a latency baseline at 384 units | +| O, per-session oracle | hot set = top K by that record's own routed mass; upper bound for any cache | new: `--cache-policy oracle` | + +Build first (half a day): the separate-copy mode, the oracle policy, expert masks combined with +`--dynamic` (N < 192), a paired bootstrap over sessions in `routed_report.py --compare A B`, and an +admission trace output for W6. Move the routed forward and `residual_cache` into a shared +`routed.py` so the pod driver can use them. + +Runs (each pass is 8 variants, about 3.5 h on the CPU): + +1. Pass 1 (chained now): N = 192; endpoints, S(96), R(96, τ 0.2 and 0.3), R(48, τ 0.2), M(τ 0.22), + per-token drop. +2. Pass 2, B = 288: S and R at N = 160 and 144, D(192, 48), O(192, 96), τ 0.15. +3. Pass 3: B = 240 and 336 with the best settings from pass 2, plus S, D and O at each. +4. Pass 4: the winners and their baselines at 4,096 tokens per record (the pod's teacher logits + cover whole records), so the cache is measured past its warm-up. + +Acceptance and scoring inside the item: + +- 10 points: R beats both S and D by at least 0.03 KL, bootstrap 95% interval excluding zero, at + one budget, with simulated SSD traffic at or below 2.5 GB/s at 30 tok/s (half the measured + sequential rate). +- 15 points: the same at two of three budgets, confirmed at 4,096 tokens. +- 25 points: reproduced on the full 384-expert model on a pod (W4b). + +W4b, pod (only after 15 points locally): fit nested codes for all 384 experts and run the same +policies with the full eval set (108 sessions). Needs a torch/CUDA backend for the 1-bit E8 search +(256 codewords, a plain argmin) and the routed forward in `collect.py`. All prep on the Mac; the +code pack uploads while the model downloads. Estimate 6–10 h on 1×H100 at $3.99/h, $25–40 plus +storage; H200 at $5.29/h if H100 stock is still out. Budget left is about $380 of $497. + +If R does not beat S, in this order: + +1. Admission by prediction instead of after use: score layer l+1's gate on layer l's output and + fetch ahead (the Fate result, 97% hit rate). Measure the hit rate offline first. +2. A stronger ladder: 2-bit base plus 1-bit residual (3 bits), budgets 384–576. The 1-bit base is + weak (error 0.31); with a 2-bit base the question becomes how close to FP4 (0.578) the routed + residual gets. +3. A long-horizon hot set (DynaExq's rule) over nested residuals, which isolates the nesting gain + from the policy. +4. If all three fail, the honest paper is the negative result plus the selection finding from + step 2c, under a different title. Say so then; do not stretch the claim. + +### W5. Unification (rubric item 5: 0 → 10) + +- One rule for all three decisions. For a slot with gate weight g on expert e in layer l, the + estimated benefit of tier t over t − 1 is g² · (ε_{t−1} − ε_t) · ‖f_e‖², with ε from the fit + metrics (ε_0 = 1) and ‖f_e‖ from the routing capture; the cost is the bytes that tier adds + (resident or read). A slot takes the highest tier whose benefit per byte is at least λ. One λ + yields per-(layer, expert) thresholds for dropping and for the residual, and the same test on + expected benefit decides which bases and residuals are resident. New `routed_policy.py` writes + the thresholds; `local_eval.py --tau-file` reads them. +- `routed_frontier.py`: KL against resident GB for static REAP + KEEP, static-nested, separate-copy + and routed, with SSD rate marked; figure and table committed. +- If the sidecar eval shows at least 0.05 KL, add sidecars as an always-resident layer tier (about + 1.1 units per layer). +- Accept when at least 12 measured configurations are on the figure and the one-λ policy is within + 0.02 KL of the best hand-tuned setting at each budget. + +### W6. Real Apple Silicon streaming (rubric item 6: 0 → 10) + +Trace-driven, since no V4.1 decoder exists on this Mac. + +- `residual_cache` writes its admission trace (record, token, layer, expert). +- `residual_stream_bench.py` replays the trace at 10, 30 and 60 tok/s against the real residual + bytes on the internal SSD: `pread` with `F_NOCACHE` into page-aligned buffers from a small thread + pool, reading each expert's extent straight out of the uncompressed npz (no second copy on + disk). Also an `mmap` + `madvise` variant. +- Measures: latency per admission, the share that lands before the next token and before that + expert is next routed, sustained GB/s, CPU use, `phys_footprint`, thermal state, bytes read per + 1,000 tokens, and the copy cost into an MLX array. +- Accept when, at the winning W4 configuration, at least 99% of admissions complete within one + token interval at 30 tok/s and the sustained rate is at most half the device's sequential rate. + Run with no other heavy job, because the fits and evals also read from this SSD. + +### W7. Learned policy and tier-aware codes (rubric item 7: 0 → 10) + +- (a) 5 points: the calibrated per-(layer, expert) thresholds from W5 beat a single global τ by at + least 0.02 KL at equal memory and SSD rate. +- (b) 5 points: tier-aware codes. Capture the importance diagonal separately for low-gate and + high-gate slots, fit the base to the low-gate statistics and the residual to the high-gate ones + (each code is fit to the tokens that use it), and gain at least 0.02 KL. Four capture chunks, + about 7 h of GPU for the refit, one eval pass. +- Not required for 100%: end-to-end distillation of thresholds and router biases with a bits + penalty. It needs backprop through the streamed model on a pod (over $100) and is worth doing + only if (a) and (b) leave a visible gap. + +### W8. Generality (rubric item 8: 0 → 10) + +- 5 points, second MoE: Qwen3-30B-A3B (specs to verify at download: 48 layers, 128 experts, top 8, + Apache-2.0, about 61 GB in BF16). It fits the Mac for the teacher pass and, unlike V4.1, can + generate locally. Build its pack with the same domains, capture routing and teacher logits, fit + nested E8 codes, port the routed forward to its MoE module, and repeat the S / R / D comparison + at three budgets. +- 5 points, downstream: on Qwen3-30B, two generative benchmarks inside the scope (coding and tool + use), chosen after a currency check so that nothing saturated is used, at least 200 items each, + static against routed at equal memory. On V4.1, likelihood-ranked tasks that fit the streamed + teacher-forced pass (ranking the correct patch or tool call among distractors) built from + held-out sources. +- Needs about 75 GB of disk and Jack's go for the download. + +### W9. Paper (rubric item 9: 1 → 5) + +- `docs/paper/` in arXiv LaTeX: method (tiers, nested lattice codes, the one-λ rule, the no-stall + cache), results (frontier, SSD traffic against τ and K, depth regimes, per-domain), related-work + table from W0, limitations (teacher-forced KL, one eval length, the tool-use anomaly). +- Sections are drafted as each workstream lands; the full draft follows W8. Publishing the nested + codes on Hugging Face needs Jack's OK. + +## 4. Order and expected score + +| When (estimate) | Work | Score after | +|---|---|---:| +| Today | W1 finishes; sidecar eval; sweep pass 1; W0 reading; W4 harness additions | 31 | +| Day 1 | W4 passes 2–3; W6 bench built and run between passes | 41–46 | +| Day 2 | W4 pass 4 (4,096 tokens); W5 policy and figure; W7a | 61–66 | +| Day 3 | W6 final numbers; W7b capture, refit, eval; W4b prep | 76 | +| Day 4 | W4b on a pod; W8 download and teacher pass | 86 | +| Days 5–6 | W8 fit, routed eval, downstream benchmarks | 96 | +| Day 7 | W9 full draft | 100 | + +The score moves only on measured results, so a failed acceptance test holds it where it is. + +## 5. Resources + +- Memory: 80 GB cap. The 8-variant routed sweep is estimated at about 65 GB; if the guard trips it + splits into two passes of four. +- Disk: 74 GB free after storing 1-bit codes as uint8. The nested fit takes 27 GB more. W8 does not + fit without freeing space. +- Pod: about $380 left. W4b is the only planned spend ($25–40). + +## 6. Decisions for Jack + +1. Disk: Jack is cleaning up himself (2026-11-09). Safe to remove: `~/Models/v41-reap75` (273 GB; + published on Hugging Face, and its experts are a subset of the local REAP-50) and + `~/Models/v41-reap-run/quant/e8p-prunedimp` (65 GB; step 2 found it changes nothing). In use, do + not remove: `~/Models/v41-reap50` and, under `~/Models/v41-reap-run/`, `stats`, `pack`, + `imatrix`, `alloc`, `evals`, `sidecar`, `jobs`, `quant/e8p-podimp`, `quant/e8-1bit-podimp` and + `quant/e8-nest-r1`. +2. Download Qwen3-30B-A3B for W8. +3. Whether the prior art in section 1 changes the title or the ambition. diff --git a/scripts/runpod/v41_reap/keep_quant.py b/scripts/runpod/v41_reap/keep_quant.py index 38c2a62..3d0f265 100644 --- a/scripts/runpod/v41_reap/keep_quant.py +++ b/scripts/runpod/v41_reap/keep_quant.py @@ -15,7 +15,7 @@ same group sizes instead (1.031 / 1.0625 bpw). ``--residual-of DIR`` fits the residual of another run's codes (``W - decode(DIR)``) instead of ``W``: a nested code, whose base is usable alone and base + residual is the higher-precision tier; ``total_wrel_mse`` scores the sum against ``W``. Output: one ``layer-XX.npz`` per layer with -``{w1,w2,w3}_codes`` uint16 ``[E, out, in/8]`` and ``{w1,w2,w3}_scales`` fp16 ``[E, out, in/g]``, +``{w1,w2,w3}_codes`` (uint16, or uint8 for 1-bit codes) ``[E, out, in/8]`` and ``{w1,w2,w3}_scales`` fp16 ``[E, out, in/g]``, plus ``metrics.jsonl``. Finished layers are skipped, so the run can resume. """ @@ -123,7 +123,8 @@ def main(argv: list[str] | None = None) -> int: code_bits=args.code_bits, iterations=args.iterations, backend=args.backend) if base is not None: total[proj].append(projection_error(w, b + reconstruct_quantized(q), d).weighted_relative_mse) - arrays.setdefault(f"{proj}_codes", []).append(np.asarray(q.codes, dtype=np.uint16)) + # 1-bit E8 codes are one byte; E8P codes are two + arrays.setdefault(f"{proj}_codes", []).append(np.asarray(q.codes, dtype=np.uint8 if args.code_bits == 8 else np.uint16)) arrays.setdefault(f"{proj}_scales", []).append(np.asarray(q.scales, dtype=np.float16)) errs[proj].append(metrics.weighted_relative_mse if hasattr(metrics, "weighted_relative_mse") else float("nan")) tmp = dest.with_suffix(".tmp.npz") From 194b02be3fd2dd65a374f079b415082401a6ffbd Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:18:05 -0800 Subject: [PATCH 30/74] docs: headline is now 'Offload the Residual, Not the Expert' --- .../2026-11-08-v41-reap-recovery-log.md | 3 +++ .../2026-11-09-routed-precision-plan.md | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 443e8eb..79b6fda 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -621,3 +621,6 @@ to 100; partial credit only for measured progress). converted in place and verified value-identical, freeing 38 GB. - The first routed sweep swaps the τ = 0.41 variant for `s96`, a static hot set of 96 residuals with no admissions: the same codes and memory as the routed cache, so the comparison isolates routing. + +- Headline changed (Jack, 2026-11-09): *Offload the Residual, Not the Expert: Unifying Pruning, Quantization and Offloading in MoE Inference*. The rubric and its weights are unchanged; item 4's + comparison against a separate-copy cache is now the title's own claim. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 00f8a33..f427404 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -1,7 +1,8 @@ # Routed precision: implementation plan to 100% -Written 2026-11-09. Target headline: *Precision as a Routed Dimension: unifying pruning, -quantization and offloading in MoE inference*. Progress is scored with the fixed rubric in the +Written 2026-11-09. Target headline (Jack, 2026-11-09, after the prior-art search below): *Offload +the Residual, Not the Expert: Unifying Pruning, Quantization and Offloading in MoE Inference*. It +replaces *Precision as a Routed Dimension*, whose first half MatMoE already covers. Progress is scored with the fixed rubric in the [recovery log](2026-11-08-v41-reap-recovery-log.md) ("Headline progress rubric"); this plan says what has to be built, run and measured for each rubric item, in what order, and what counts as done. Score today: **26 / 100**. @@ -35,8 +36,8 @@ What none of them appears to do, and what the paper therefore has to stand on: 3. **The sub-2-bit lattice regime** (1 and 2 bits per weight, E8 codes), on a 384-expert, top-6 frontier model, on a 128 GB laptop. -The headline can stay, because "unifying pruning, quantization and offloading" is the surviving -claim. The first half of it is no longer new on its own, and the paper must cite and beat +The new title leads with claim 1 and keeps "unifying pruning, quantization and offloading", the +half of the original headline that survives. The paper must cite and beat MatMoE-style and DynaExq-style policies rather than only static baselines. If W0 finds a paper that already offloads nested residuals under per-token tiers, the paper becomes an empirical study of that design at 2 bits and below, and Jack decides whether to continue. From 264190b9d1d79849f4ef2cc6628bc35369e2a8b8 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:20:33 -0800 Subject: [PATCH 31/74] docs: candidate levers X1-X7 from the campaign and the KEEP survey --- .../2026-11-09-routed-precision-plan.md | 34 +++++++++++++++++++ scripts/runpod/v41_reap/routed_report.py | 6 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index f427404..9a32b15 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -234,3 +234,37 @@ The score moves only on measured results, so a failed acceptance test holds it w `quant/e8-nest-r1`. 2. Download Qwen3-30B-A3B for W8. 3. Whether the prior art in section 1 changes the title or the ambition. + +## 7. Candidate levers beyond the prior art (2026-11-09) + +From this campaign's results and a survey of the older KEEP documents. None is checked against the +literature yet; each gets a search before it is claimed. Ordered by expected value. + +| # | Lever | Evidence in hand | Test | Where it runs | +|---|---|---|---|---| +| X1 | Prune twice at 384 → 192. REAP-50 was chosen on full-model routing with the per-route score; its KL (0.578) is the floor under everything here. | The same fix cut KL 15% at 192 → 96 (step 2c ablation); a second pass saturates (step 2d). | Calibration pass on the full model masked to 288 by total mass, re-rank to 192, eval against shipped REAP-50. | Pod, with W4b; about $15–25 extra | +| X2 | Calibration-targeted pruning as specialization. | REAP-50 beats the full model by 1.22 nats/token on tool-use NLL; the pod's own student shows the same gap. Logged as an anomaly, never chased. | Per-position NLL of teacher and students on tool-use with the full model resident; then a downstream tool-use benchmark (W8). | Pod, same run as X1 | +| X3 | Keep every expert, offload residuals, instead of pruning half. | V4.1 plan: each expert kept costs 0.359 GB of RAM at 2 bits "but adds no per-token reads", and the fit "forces a ~50% prune". All 384 one-bit bases cost the same memory as REAP-50 at 2 bits. | 384 bases + K residuals against REAP-50 at 2 bits (0.747) at equal resident memory. This is the title's claim at full scale. | Pod, W4b; the Mac sweep at 192 vs 96 is its proxy | +| X4 | The OS page cache as the precision cache. Residuals are memory-mapped clean file pages; `mincore` asks whether one is resident, `madvise(WILLNEED)` requests it in the background, and the kernel evicts under pressure at no write cost. Precision degrades instead of the machine swapping. | Roadmap, 2026-08-21: at 90.99 GB of MLX-active memory 0 of 5 timing pairs were clean and the bind alone logged 535,792 swapouts. Anonymous memory at that footprint thrashes; clean file pages cannot. | In W6: replay the trace through `mmap` + `mincore`, with and without a memory hog, and measure hits, latency and swapouts. | Mac | +| X5 | A stronger base and finer rungs. The 1-bit base is the weak link. | Pilot: 1-bit codes gave a 0.56 block cosine; the SwiGLU amplifies projection error about 2.8×; 2 bits is "at its rate-distortion limit". Here, a static 1-bit tier added 0.025 KL at best. | (a) Projection-level tiers from the existing codes: residual for the down projection only (1.33 bits) or gate/up only (1.67 bits), no new fit. (b) The 2-bit base + 1-bit residual ladder, moved up from fallback to W4 pass 3. | Mac | +| X6 | Budget by sensitivity, not routed mass. | Step 1: mass-based reallocation lost to uniform (1.046 vs 0.955 at 128) although layers 20–39 are far more concentrated than 0–19. | Mask a fixed share of experts in one layer band at a time and measure KL; allocate by measured sensitivity. Two eval passes. | Mac, after W4 | +| X7 | A shared base across similar experts (base = one matrix per cluster, residual per expert). | None; speculative. | Functional similarity of experts on shared inputs for 3 layers; stop if experts are near-orthogonal. | Mac, about 1 h | + +Smaller findings worth a paragraph in the paper or a note of their own: + +- Expert precision is a memory lever, not a speed lever, on this family: on V4-Flash, routed experts + were 11% of the 15.27 GB read per decoded token and attention 67% (roadmap, 2026-08-21). MatMoE's + latency argument for dynamic precision does not carry over; the memory argument does. The + `tok_s_bound` column of `routed_report.py` counts expert bytes only and is not a speed estimate. +- Incoherence processing (the random rotation QuIP# relies on) made importance-aware E8P fits 15% + worse on the down projection, because it flattens a per-column importance that spans 180× + (wave-5 pilot). +- A gate/up-swapped artifact loads, gives finite logits and scores cosine 0.9993; only a + discrimination ratio caught it. Cosine is not a correctness check. +- Negative priors for the old recovery steps 4–5: on GLM-4.5-Air router distillation was rejected + (top-1 change slightly negative) and a rank-4 sidecar improved top-1 from 0.795 to 0.852 but ran + 1.72× slower and failed the speed gate. Sidecars here are judged on quality per unit of memory and + on their per-token cost, and router distillation stays off the critical path. +- A quality reference from outside: the Rapid-MLX V4.1 build is reported at KL 0.26 and top-1 85% on + a 256 GB M3 Ultra (local test plan, 2026-11-08). Our best 2-bit REAP-50 number is 0.747. A result + that impresses has to close most of that gap at about half the memory. diff --git a/scripts/runpod/v41_reap/routed_report.py b/scripts/runpod/v41_reap/routed_report.py index 6b5266c..96700f1 100644 --- a/scripts/runpod/v41_reap/routed_report.py +++ b/scripts/runpod/v41_reap/routed_report.py @@ -4,8 +4,10 @@ - resident expert memory: every base (tier 1) plus K residuals per layer when cached, otherwise every residual too (nothing streamed), in units per layer (1 unit = one 1-bit expert) and GB. - SSD traffic: residual admissions per token, in MB/token and GB/s at ``--tok-s``. -- unified-memory bytes read per token by routed experts (tier 1 = base, tier 2 = base + residual), - the decode bound at ``--bandwidth-gbs``. +- unified-memory bytes read per token by routed experts (tier 1 = base, tier 2 = base + residual). + ``tok_s_bound`` divides ``--bandwidth-gbs`` by those bytes alone; routed experts were about 11% + of decode traffic on V4-Flash, so it is an upper bound on what expert tiers can buy, not a speed + estimate. python routed_report.py --eval ~/Models/v41-reap-run/evals/step4-routed [--eval ...] """ From cddc65a7b34ea4d45332f8b4718271d1227a3e27 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:25:33 -0800 Subject: [PATCH 32/74] v41 routed precision: separate-copy and oracle baselines, projection rungs, traces, bootstrap; findings survey and levers --- .../2026-11-08-v41-reap-recovery-log.md | 9 + .../2026-11-09-keep-findings-and-levers.md | 166 ++++++++++++++++++ .../2026-11-09-routed-precision-plan.md | 34 ++-- scripts/runpod/v41_reap/local_eval.py | 56 +++++- scripts/runpod/v41_reap/routed_report.py | 67 ++++++- tests/test_runpod_v41_routed_precision.py | 28 +++ 6 files changed, 334 insertions(+), 26 deletions(-) create mode 100644 docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 79b6fda..d04d18c 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -624,3 +624,12 @@ to 100; partial credit only for measured progress). - Headline changed (Jack, 2026-11-09): *Offload the Residual, Not the Expert: Unifying Pruning, Quantization and Offloading in MoE Inference*. The rubric and its weights are unchanged; item 4's comparison against a separate-copy cache is now the title's own claim. +- Survey of the older KEEP documents, this campaign's findings with bootstrap intervals, and the + candidate levers X1–X7: [2026-11-09-keep-findings-and-levers.md](2026-11-09-keep-findings-and-levers.md). + Bootstrap (positions, paired): selection on pruned routing −0.177 KL [−0.200, −0.148]; second + prune pass −0.024 [−0.034, −0.012]; static allocator at 288 against 144 at 2 bits −0.003 + [−0.009, +0.005]. +- Harness additions for W4: separate-copy high tier (`--dynamic BASE|HI`), oracle cache policy, + projection-level rungs (`--hi-proj`), admission traces (`--trace`), expert masks under + `--dynamic`, and `routed_report.py --compare`. Passes 2, 2b and 3 are queued behind pass 1 + (`~/Models/v41-reap-run/jobs/queue2.sh`, 24 variants, about 11 h). diff --git a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md new file mode 100644 index 0000000..58adfb6 --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md @@ -0,0 +1,166 @@ +# What KEEP has learned, and which of it could go beyond the prior art + +Written 2026-11-09 for the paper track (*Offload the Residual, Not the Expert*). Jack asked what the +whole project has measured that may not be in the literature, and whether any win is a bigger lever +than it looked. Three sources: this campaign's own results +([recovery log](2026-11-08-v41-reap-recovery-log.md)), a survey of the older KEEP documents, and the +prior-art search in the [plan](2026-11-09-routed-precision-plan.md). The scheduled experiments (X1–X7) +are in section 7 of the plan; this file holds the evidence and the reasoning. + +Nothing here has been checked against the literature beyond the six papers in the plan. Each lever +gets its own search before it is claimed. + +## 1. Findings from this campaign + +Differences are domain-mean KL on the Mac eval set (22 sessions, first 1,024 tokens), with a paired +block bootstrap over positions (`routed_report.py --compare`). The interval covers position noise +within the 22 sessions, not which sessions were drawn. + +| Finding | Number | 95% interval | +|---|---:|---| +| Choosing 96 experts by total routed mass on the pruned model's routing, against REAP-75's set, both at 2 bits | −0.177 KL | −0.200 to −0.148 | +| A second prune pass (192 → 144 → 96) over one pass | −0.024 | −0.034 to −0.012 | +| Static allocator at 288 units against 144 experts at 2 bits | −0.003 | −0.009 to +0.005 | + +- **Selection on the pruned model's routing is the lever; precision allocation was not.** Each of the + two changes alone (the ranking rule, or measuring on the pruned model) gave about 0.025. Together + 0.177. A second pass is statistically real but small, most of it on tool-use. +- **Quantization statistics survive pruning; routing does not.** Refitting the 2-bit codes with + importance measured on the pruned model changed nothing (0.753 against 0.747). The importance + cosine was 0.976; the selection overlap with REAP-75 was 58%. +- **Losses are not additive.** KEEP 2-bit costs +0.17 KL on REAP-50 and +0.02 on a 96-expert model. +- **Two regimes by depth.** The top 96 experts hold 72–83% of routed mass in layers 0–19 and 94–98% + in layers 20–39. Yet moving budget toward the diffuse layers lost to a uniform budget (1.046 + against 0.955 at 128 experts): concentration is not sensitivity. +- **A static 1-bit tier is nearly worthless** (0.025 KL at 192 units, nothing at 288). The 1-bit + code's error is 0.31 per projection. +- **Nested lattice codes cost about 12%.** A 1-bit E8 residual on a 1-bit E8 base scores 0.101 + (gate/up) against 0.090 for a flat 2-bit E8P fit. +- **The pruned model beats the full model on tool-use**, by 1.22 nats per token of NLL, on the pod's + own student as well as locally. Not explained. +- **Sidecars (running):** a rank-256 linear correction removes 36–76% of the layer gap where the + gap is small (layers 0–3) and 9–15% where it is large (layers 4–6). + +## 2. Findings from the older KEEP documents + +From a survey of every document under `docs/` and the module docstrings. Rows marked ✓ were +re-read at the cited line for this note; the rest are as the survey reported them. + +### Memory and speed come from different weights + +- ✓ On V4-Flash, decoding one token reads 15.27 GB: attention 67%, routed experts 11% + (`2026-08-21-roadmap.md:116-121`). Residents are dequantized to bf16 at bind. +- ✓ On V4.1, each expert kept in every layer costs 0.359 GB of RAM at 2.031 bits per weight "but adds + no per-token" reads, and fitting the Mac "forces a ~50% prune" + (`2026-11-05-v41-plan-and-phase0-status.md:15-21`). +- GLM-5.3 inverts this: residents are about 85% of decode traffic (`glm-5.3-flash/2026-10-06-plan.md`). + +So on the DeepSeek family, expert precision buys memory and almost no speed. MatMoE argues dynamic +precision from latency; that argument does not carry to this architecture, and the memory argument +is the one this paper makes. + +### The Mac thrashes at a 91 GB resident set + +- ✓ At 90.99 GB of MLX-active memory the bind alone logged 535,792 swapouts (8.4 GB of anonymous + buffers), and byte-identical work spanned 16.5× in time across rows that all passed the quiet + gate (`2026-08-21-roadmap.md:84-89, 208-232`). The document's own conclusion is that the timings + were "memory weather, not decode". +- Policy since then: leave the wired limit at its default (`docs/troubleshooting.md`). + +Anonymous memory at this footprint gets compressed and swapped, which writes to the SSD and stalls +decode. Clean file-backed pages cannot be swapped; they are dropped and read again. That is the +case for lever X4. + +### The quantizer + +- ✓ A 1-bit code gave a 0.56 block cosine, "not a usable model"; the SwiGLU amplifies projection error + about 2.8× at block level; E8P beats affine quantization below about 3.1 bits per weight and + loses above; 2 bits is "at its rate-distortion limit" (`research/wave5-pilot-report.md:12-14, 122, + 215, 305, 339`). +- ✓ Incoherence processing (the random rotation QuIP# is built on) made the importance-aware fit 15% + worse on the down projection, 9.18e-2 against 7.97e-2, because the rotation flattens a per-column + importance that spans 180× (`research/wave5-pilot-report.md:144-149`). +- The codebook step returns 0.71 of fractional error per bit per weight and group-size steps + 0.17–0.24, so group size is the rate dial. Uniform group size beats non-uniform at matched rate. + The belief that the down projection needs extra protection was not supported. +- Only 1 of 11,008 (layer, expert) pairs had no calibration routes. +- An fp32 reduction flips near-tie argmins in about 30% of scale last bits, so exact fits keep + float64 sums on the CPU. + +### Checks that lie + +- ✓ A gate/up-swapped artifact loads, gives finite logits and scores cosine 0.9993 against the correct + one; only a discrimination ratio (4.3e10) caught it (`research/wave5-materializer-report.md:12, + 101`). +- A per-call-sync benchmark harness produced false kills: the E8P kernel read 52.8 GB/s and was + marked KILL, then 122–126 GB/s chained and 201.9–205.5 GB/s tuned (`2026-11-05-phase0-kill-tests.md`). +- Two "slower than source" verdicts were bugs in the measuring path: a Python-version mismatch that + made about 50 kernels unreachable (8.36× once fixed), and a loader that re-globbed the build tree + on every call (4.53× once fixed) (`handoff.md`, `2026-08-21-compounded-speed-verdict.md`). + +### Recovery levers already tried on GLM-4.5-Air + +- ✓ Router distillation was rejected: best global top-1 change "slightly negative" + (`glm45-air/rc-pipeline.md:176`). +- ✓ A rank-4 sidecar raised report top-1 from 0.795 to 0.852 and cut KL to 0.314, but ran 1.718× the + control and failed the speed gate (`glm45-air/rc-pipeline.md:167, 895, 930`). A proxy win that + failed end to end. +- Raising non-expert precision changed nothing once a loader bug was fixed. + +### Streaming and I/O + +- A layer-outer, chunk-inner teacher loop is bit-identical to chunk-major and avoided 58.8 h of I/O. +- Prefill I/O favoured VQ: 6 `preadv` calls per layer against 1,536; the gap to the source model was + compute, not I/O. +- Engram lookups depend only on token ids; the tables are memory-mapped and gathered on the host. + The V4.1 plan already keeps Engram's 203 GB on the SSD. +- ✓ A reference point from outside: the Rapid-MLX V4.1 build reports teacher-forced KL 0.26 and top-1 + 85% against the BF16 source, at 34–49 tok/s on a 256 GB M3 Ultra + (`2026-11-08-v41-reap75-local-test-plan.md:125-126`). + +## 3. What looks like it could go beyond the prior art + +In the order I would bet on them. X numbers refer to the plan. + +1. **Prune twice, at the first cut (X1).** The one clearly significant win of this campaign has only + been applied to the second halving. REAP-50's own KL (0.578) is the floor under every number + here, and it was chosen the old way. If the effect repeats at 384 → 192 it lifts the whole + frontier, and "measure routing on the pruned model, rank by total mass" is a two-line change to + REAP that anyone can use. Needs the full model: a pod run. +2. **Keep every expert and page residuals instead of pruning half (X3).** The V4.1 plan prunes 50% + because each kept expert costs 0.359 GB and the model has to fit. All 384 one-bit bases cost the + same memory as 192 experts at 2 bits. Whether 384 bases plus a residual cache beats REAP-50 at + 2 bits at equal memory is the title's claim at full scale, and no paper found so far compares + pruning against residual offload. The local sweep (192 bases against 96 or 144 at 2 bits) is its + proxy; the weak 1-bit base is the risk. +3. **The OS page cache as the precision cache (X4).** Wire the bases; map the residuals as clean + file pages; `mincore` before use, `madvise` to fetch. No cache code, and under memory pressure the + model loses precision instead of swapping. KEEP has already measured the failure this avoids + (535,792 swapouts, 16.5× timing spread at 91 GB). This is specific to how unified memory and the + macOS VM behave, and it turns "SSD churn" from writes into reads. +4. **Pruning as specialization (X2).** If the tool-use result holds up against the resident full + model and on a downstream benchmark, a calibration-targeted prune is not only smaller but better + on its target domains. Large if true; the prior is that it is partly a teacher artifact. +5. **One artifact for every budget.** Because quantization statistics survive pruning and one mass + ranking yields nested expert sets (pm-144 ⊂ pm-160; 97% overlap between passes), a single set of + files can serve any memory budget: nested in expert count and in bits. MatMoE nests bits only. + This is a packaging result that follows from 1–3 rather than an experiment of its own. +6. **Finer rungs and a stronger base (X5).** The 1-bit base is the weak link by three separate + measurements. Projection-level residuals give 1.33- and 1.67-bit rungs from the existing codes, + and a 2-bit base with a 1-bit residual moves the whole ladder to where the base is usable. +7. **Sensitivity-based budgets (X6)** and **a shared base across experts (X7)** are cheaper to test + than to argue about. + +Smaller results that are probably worth a paragraph or a short note each: incoherence processing +hurting importance-aware lattice fits; cosine similarity passing a gate/up-swapped model; and the +measurement lessons (per-call sync, one-sided paging gates). + +## 4. A correction to my own earlier claim + +I first wrote that a nested cache "holds twice as many hot experts" as a separate-copy cache at +equal memory. That is true only if the separate-copy cache keeps the low-precision copy of every hot +expert. One that drops it and reads it back on demotion uses the same memory as the nested design; +what it pays is traffic, three units per promote/demote cycle against one, and a slightly more +accurate high tier in return (0.090 against 0.101). The comparison is therefore run both ways, and +at equal SSD rate as well as equal memory. The title's claim may turn out to be about traffic and +simplicity more than about memory. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 9a32b15..7e89e22 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -28,9 +28,15 @@ What none of them appears to do, and what the paper therefore has to stand on: 1. **The offloaded unit is the nested residual.** Tiers span unified memory and SSD inside one code: the base is always resident, and a promotion reads only the residual (4.6 MB), never a - second full copy. Separate-copy systems (DynaExq, HOBBIT) pay 2 extra units of memory per hot - expert and 9.2 MB per promotion; at equal memory a nested design holds twice as many hot - experts. This is directly measurable in our harness. + second full copy. What that buys depends on how a separate-copy cache (DynaExq, HOBBIT) is run, + and both readings are measured (corrected 2026-11-09; the first draft claimed only the first): + - if it keeps the low copy of a hot expert, a hot expert costs 2 extra units and a nested design + holds twice as many hot experts at equal memory; + - if it drops the low copy and reads it back on demotion, memory is equal, but every + promote/demote cycle moves 3 units (2 in, 1 back) where a residual moves 1, and a demotion is + free. At an equal SSD rate the nested cache can admit three times as often. + The separate 2-bit copy is slightly more accurate than base + residual (error 0.090 against + 0.101), so the nested design has to win on memory or on traffic, not on per-expert accuracy. 2. **Tier 0 in the same policy.** Expert pruning (static tier 0), per-token slot dropping, precision and residency all come from one rule with one trade-off parameter. 3. **The sub-2-bit lattice regime** (1 and 2 bits per weight, E8 codes), on a 384-expert, top-6 @@ -92,7 +98,7 @@ the same codes: |---|---|---| | S, static-nested | hot set = top K by pruned-model routed mass, never changes | `--cache-k K --tau 1e9` with `--cache-warm` | | R, routed-nested (ours) | LRU of K residuals, async admission at gate ≥ τ | `--cache-k K --tau τ` | -| D, separate-copy cache (DynaExq / HOBBIT style) | high tier is a second flat 2-bit copy, so a hot expert costs 2 extra units and a promotion reads 9.2 MB; K = (B − N) / 2 | new: `--dynamic BASE\|HI` | +| D, separate-copy cache (DynaExq / HOBBIT style) | high tier is a second flat 2-bit copy. Scored two ways from one run: keeping the low copy (N + 2K units, 2 units per fetch) or swapping it (N + K units, 3 units per cycle) | `--dynamic BASE\|HI`; `routed_report.py` prints both | | M, all-resident routed (MatMoE style) | every residual resident, tier by routing score | `--tau τ`, no cache; a latency baseline at 384 units | | O, per-session oracle | hot set = top K by that record's own routed mass; upper bound for any cache | new: `--cache-policy oracle` | @@ -103,18 +109,22 @@ admission trace output for W6. Move the routed forward and `residual_cache` into Runs (each pass is 8 variants, about 3.5 h on the CPU): -1. Pass 1 (chained now): N = 192; endpoints, S(96), R(96, τ 0.2 and 0.3), R(48, τ 0.2), M(τ 0.22), +1. Pass 1 (chained): N = 192; endpoints, S(96), R(96, τ 0.2 and 0.3), R(48, τ 0.2), M(τ 0.22), per-token drop. -2. Pass 2, B = 288: S and R at N = 160 and 144, D(192, 48), O(192, 96), τ 0.15. -3. Pass 3: B = 240 and 336 with the best settings from pass 2, plus S, D and O at each. -4. Pass 4: the winners and their baselines at 4,096 tokens per record (the pod's teacher logits +2. Pass 2 (queued): O(96), R(96, τ 0.15), N = 144 all at tier 2, N = 160 with K = 128 routed and + static, S(48), and the two projection-level rungs (X5a). +3. Pass 2b (queued), separate copies: K = 96 at τ 0.2, 0.3 and 0.4 and static; K = 48 and 144. +4. Pass 3 (queued), budgets 240 and 336: R, S and O at K = 144; R(48, τ 0.3), O(48); N = 160 with + K = 80 routed and static. +5. Pass 4: the winners and their baselines at 4,096 tokens per record (the pod's teacher logits cover whole records), so the cache is measured past its warm-up. Acceptance and scoring inside the item: -- 10 points: R beats both S and D by at least 0.03 KL, bootstrap 95% interval excluding zero, at - one budget, with simulated SSD traffic at or below 2.5 GB/s at 30 tok/s (half the measured - sequential rate). +- 10 points: R beats S by at least 0.03 KL, and beats D at equal memory and equal SSD rate (under + whichever of D's two accountings is better for D), bootstrap 95% interval excluding zero, at one + budget, with simulated SSD traffic at or below 2.5 GB/s at 30 tok/s (half the measured sequential + rate). - 15 points: the same at two of three budgets, confirmed at 4,096 tokens. - 25 points: reproduced on the full 384-expert model on a pod (W4b). @@ -245,7 +255,7 @@ literature yet; each gets a search before it is claimed. Ordered by expected val | X1 | Prune twice at 384 → 192. REAP-50 was chosen on full-model routing with the per-route score; its KL (0.578) is the floor under everything here. | The same fix cut KL 15% at 192 → 96 (step 2c ablation); a second pass saturates (step 2d). | Calibration pass on the full model masked to 288 by total mass, re-rank to 192, eval against shipped REAP-50. | Pod, with W4b; about $15–25 extra | | X2 | Calibration-targeted pruning as specialization. | REAP-50 beats the full model by 1.22 nats/token on tool-use NLL; the pod's own student shows the same gap. Logged as an anomaly, never chased. | Per-position NLL of teacher and students on tool-use with the full model resident; then a downstream tool-use benchmark (W8). | Pod, same run as X1 | | X3 | Keep every expert, offload residuals, instead of pruning half. | V4.1 plan: each expert kept costs 0.359 GB of RAM at 2 bits "but adds no per-token reads", and the fit "forces a ~50% prune". All 384 one-bit bases cost the same memory as REAP-50 at 2 bits. | 384 bases + K residuals against REAP-50 at 2 bits (0.747) at equal resident memory. This is the title's claim at full scale. | Pod, W4b; the Mac sweep at 192 vs 96 is its proxy | -| X4 | The OS page cache as the precision cache. Residuals are memory-mapped clean file pages; `mincore` asks whether one is resident, `madvise(WILLNEED)` requests it in the background, and the kernel evicts under pressure at no write cost. Precision degrades instead of the machine swapping. | Roadmap, 2026-08-21: at 90.99 GB of MLX-active memory 0 of 5 timing pairs were clean and the bind alone logged 535,792 swapouts. Anonymous memory at that footprint thrashes; clean file pages cannot. | In W6: replay the trace through `mmap` + `mincore`, with and without a memory hog, and measure hits, latency and swapouts. | Mac | +| X4 | The OS page cache as the precision cache. Residuals are memory-mapped clean file pages; `mincore` asks whether one is resident, `madvise(WILLNEED)` requests it in the background, and the kernel evicts under pressure at no write cost. Precision degrades instead of the machine swapping. | Roadmap, 2026-08-21: at 90.99 GB of MLX-active memory byte-identical work spanned 16.5× in time and the bind alone logged 535,792 swapouts. Anonymous memory at that footprint thrashes; clean file pages cannot. | In W6: replay the trace through `mmap` + `mincore`, with and without a memory hog, and measure hits, latency and swapouts. | Mac | | X5 | A stronger base and finer rungs. The 1-bit base is the weak link. | Pilot: 1-bit codes gave a 0.56 block cosine; the SwiGLU amplifies projection error about 2.8×; 2 bits is "at its rate-distortion limit". Here, a static 1-bit tier added 0.025 KL at best. | (a) Projection-level tiers from the existing codes: residual for the down projection only (1.33 bits) or gate/up only (1.67 bits), no new fit. (b) The 2-bit base + 1-bit residual ladder, moved up from fallback to W4 pass 3. | Mac | | X6 | Budget by sensitivity, not routed mass. | Step 1: mass-based reallocation lost to uniform (1.046 vs 0.955 at 128) although layers 20–39 are far more concentrated than 0–19. | Mask a fixed share of experts in one layer band at a time and measure KL; allocate by measured sensitivity. Two eval passes. | Mac, after W4 | | X7 | A shared base across similar experts (base = one matrix per cluster, residual per expert). | None; speculative. | Functional similarity of experts on shared inputs for 3 layers; stop if experts are near-orthogonal. | Mac, about 1 h | diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index fa34a9a..2046a10 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -127,20 +127,31 @@ def mixed(layer: int, sd: dict) -> dict: def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: list[int], - warm: np.ndarray | None, counts: np.ndarray) -> np.ndarray: + warm: np.ndarray | None, counts: np.ndarray, policy: str = "lru", + trace: list | None = None) -> np.ndarray: """Which (token, slot) pairs may run base + residual, under an LRU of ``k`` resident residuals. ``ind``/``wts``: [T, top_k] routed experts and gate weights, the records of ``bounds`` back to back. A slot runs high precision only if its expert's residual is already resident. After each token, every slot with gate >= ``tau`` admits (or refreshes) its residual: an async SSD fetch that serves the next token on, so decode never stalls. Each record starts from ``warm`` (or - empty). ``counts[e, 4]`` accumulates admissions, i.e. residual fetches.""" + empty). ``counts[e, 4]`` accumulates admissions, i.e. residual fetches; ``trace`` collects + them as (record, token in record, expert). + + ``policy="oracle"`` is the upper bound for any cache of ``k``: each record's hot set is its own + top ``k`` experts by routed gate weight, known in advance and never changed (no admissions).""" from collections import OrderedDict hi = np.zeros(ind.shape, dtype=bool) start = 0 - for n in bounds: + for rec, n in enumerate(bounds): + if policy == "oracle": + mass = np.bincount(ind[start:start + n].ravel(), weights=wts[start:start + n].ravel(), minlength=counts.shape[0]) + hot = np.argsort(-mass, kind="stable")[:k] + hi[start:start + n] = np.isin(ind[start:start + n], hot[mass[hot] > 0]) + start += n + continue lru = OrderedDict((int(e), None) for e in (warm if warm is not None else [])) for t in range(start, start + n): row = ind[t].tolist() @@ -151,6 +162,8 @@ def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: lru.move_to_end(e) else: counts[e, 4] += 1 + if trace is not None: + trace.append((rec, t - start, e)) lru[e] = None if len(lru) > k: lru.popitem(last=False) @@ -172,7 +185,14 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--threads", type=int, default=os.cpu_count()) p.add_argument("--expert-override", help="'fp4-bf16' or a keep_quant.py output directory") p.add_argument("--capture-imatrix", type=Path, help="measure the importance diagonal instead of scoring") - p.add_argument("--dynamic", help="BASE_DIR+RESIDUAL_DIR: per-token tiers, base alone or base + nested residual") + p.add_argument("--dynamic", help="per-token tiers. BASE_DIR+RESIDUAL_DIR: base alone or base + nested residual. " + "BASE_DIR|HI_DIR: base alone or a separate high-precision copy (a hot expert then holds both)") + p.add_argument("--hi-proj", action="append", default=[], metavar="NAME=w2|w1w3", + help="nested only, per variant: the high tier adds the residual for these projections only") + p.add_argument("--cache-policy", action="append", default=[], metavar="NAME=lru|oracle", + help="with --cache-k: lru (default) or oracle (each record's own top-K, the upper bound)") + p.add_argument("--trace", action="append", default=[], metavar="NAME", + help="with --cache-k: write this variant's residual admissions (layer, record, token, expert)") p.add_argument("--tau", action="append", default=[], metavar="NAME=VALUE", help="with --dynamic, per variant: a slot runs base + residual when its gate weight >= VALUE") p.add_argument("--tau-drop", action="append", default=[], metavar="NAME=VALUE", @@ -238,8 +258,13 @@ def main(argv: list[str] | None = None) -> int: from keep.vq.e8 import e8_1bit_grid, e8p_full_grid - base_dir, _, res_dir = args.dynamic.partition("+") + copy_mode = "|" in args.dynamic # the high tier is its own copy, not base + residual + base_dir, _, res_dir = args.dynamic.partition("|" if copy_mode else "+") base_dir, res_dir = Path(base_dir).expanduser(), Path(res_dir).expanduser() + hi_proj = {name: {"w2": ("w2",), "w1w3": ("w1", "w3")}[v] for name, _, v in (s.partition("=") for s in args.hi_proj)} + assert not (copy_mode and hi_proj), "--hi-proj needs nested codes" + policies = {name: v for name, _, v in (s.partition("=") for s in args.cache_policy)} + traces: dict[str, list] = {name: [] for name in args.trace} taus = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau)} tau_drop = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau_drop)} assert set(taus) == set(variants), "--tau needs one NAME=VALUE per --variant" @@ -254,7 +279,12 @@ def main(argv: list[str] | None = None) -> int: dyn_counts[v] = np.zeros((n_layers_, n_local, 7)) def simulate_cache(layer, ind, wts, k, tau, counts): - return residual_cache(ind, wts, k, tau, state["bounds"], None if warm is None else warm[layer, :k], counts) + v, found = state["variant"], [] + hi = residual_cache(ind, wts, k, tau, state["bounds"], None if warm is None else warm[layer, :k], counts, + policy=policies.get(v, "lru"), trace=found if v in traces else None) + if v in traces: + traces[v].extend((layer, *row) for row in found) + return hi def tier_weights(layer: int, e: int) -> tuple[dict, dict]: """(base, base + residual) bf16 weights of local expert ``e``, decoded once per layer.""" @@ -274,7 +304,7 @@ def tier_weights(layer: int, e: int) -> tuple[dict, dict]: tier[proj] = vec.reshape(vec.shape[0], -1) tiers.append(tier) lo = {p: torch.as_tensor(w).bfloat16() for p, w in tiers[0].items()} - hi = {p: torch.as_tensor(tiers[0][p] + tiers[1][p]).bfloat16() for p in tiers[0]} + hi = {p: torch.as_tensor(tiers[1][p] if copy_mode else tiers[0][p] + tiers[1][p]).bfloat16() for p in tiers[0]} cache[e] = (lo, hi) return cache[e] @@ -303,6 +333,9 @@ def dynamic_forward(self, x, image_mask=None): counts[i, 6] += float(w[drop].sum()) hi_mask = hi_mask & ~drop lo_w, hi_w = tier_weights(layer, i) + only = hi_proj.get(state["variant"]) + if only is not None: # a finer rung: the residual of some projections only + hi_w = {p: hi_w[p] if p in only else lo_w[p] for p in lo_w} for mask, tier, col in ((~hi_mask & ~drop, lo_w, 0), (hi_mask, hi_w, 1)): if mask.any(): y[idx[mask]] += run_expert(x[idx[mask]], w[mask, None], tier, limit).float() @@ -441,7 +474,16 @@ def before_moe(layer: int, block, key) -> None: for v, c in dyn_counts.items(): # per-token precision: how often (and with how much gate mass) the hi tier ran np.save(args.out / v / "tier-counts.npy", c) tot = c.sum(axis=(0, 1)) + if v in traces: + t = np.asarray(traces[v], dtype=np.int32).reshape(-1, 4) + np.savez(args.out / v / "admissions.npz", layer=t[:, 0], record=t[:, 1], token=t[:, 2], expert=t[:, 3], + records=np.asarray(list(records)), tokens=np.asarray(state["bounds"])) + kept = n_local if variants[v] is None else float(np.mean([int((~m).sum()) for m in variants[v].values()])) + # hi_units: memory (and bytes fetched) the high tier adds per expert, in units of one 1-bit expert + hi_units = 2.0 if copy_mode else len(hi_proj.get(v, ("w1", "w2", "w3"))) / 3 summary["variants"][v]["_tiers"] = {"tau": taus[v], "cache_k": cache_k.get(v), + "policy": policies.get(v, "lru") if v in cache_k else None, + "copy_mode": copy_mode, "hi_units": hi_units, "n_bases": kept, "tau_drop": tau_drop.get(v, 0.0), "hi_slot_share": float(tot[1] / (tot[:2].sum() + tot[5])), "drop_slot_share": float(tot[5] / (tot[:2].sum() + tot[5])), diff --git a/scripts/runpod/v41_reap/routed_report.py b/scripts/runpod/v41_reap/routed_report.py index 96700f1..8f58921 100644 --- a/scripts/runpod/v41_reap/routed_report.py +++ b/scripts/runpod/v41_reap/routed_report.py @@ -10,6 +10,7 @@ estimate. python routed_report.py --eval ~/Models/v41-reap-run/evals/step4-routed [--eval ...] + python routed_report.py --compare EVAL_DIR:c96t2 EVAL_DIR:s96 """ from __future__ import annotations @@ -35,17 +36,25 @@ def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: out.append(row) continue c = np.load(eval_dir / name / "tier-counts.npy") # [L, E, 7] - n_layers, n_exp = c.shape[:2] + n_layers = c.shape[0] used = c[:, :, :2].sum() + c[:, :, 5].sum() # every routed slot tokens = used / 6 / n_layers k = tiers.get("cache_k") - units = n_exp + (k if k is not None else n_exp) - read = (c[:, :, 0].sum() * UNIT_BYTES + c[:, :, 1].sum() * 2 * UNIT_BYTES) / tokens - ssd = c[:, :, 4].sum() * UNIT_BYTES / tokens if k is not None else 0.0 + n_bases, hi_units = tiers.get("n_bases", c.shape[1]), tiers.get("hi_units", 1.0) + units = n_bases + (k if k is not None else n_bases) * hi_units + hi_read = hi_units if tiers.get("copy_mode") else 1 + hi_units # a separate copy is read alone + read = (c[:, :, 0].sum() + c[:, :, 1].sum() * hi_read) * UNIT_BYTES / tokens + ssd = c[:, :, 4].sum() * hi_units * UNIT_BYTES / tokens if k is not None else 0.0 + if tiers.get("copy_mode") and k is not None: + # A separate-copy cache can instead drop a hot expert's low copy (1 extra unit, not 2) and read + # it back on demotion: every promote/demote cycle then moves 3 units where a residual moves 1. + row |= {"units_per_layer_swap": round(n_bases + k, 1), + "ssd_gbs_swap": round(c[:, :, 4].sum() * 3 * UNIT_BYTES / tokens * tok_s / 1e9, 2)} row |= { - "tau": tiers["tau"], "tau_drop": tiers.get("tau_drop", 0.0), "cache_k": k, + "tau": tiers["tau"], "tau_drop": tiers.get("tau_drop", 0.0), "cache_k": k, "policy": tiers.get("policy"), + "n_bases": n_bases, "hi_units": round(hi_units, 2), "hi_slots": round(tiers["hi_slot_share"], 3), "drop_slots": round(tiers.get("drop_slot_share", 0.0), 3), - "units_per_layer": units, "resident_gb": round(units * n_layers * UNIT_BYTES / 1e9, 1), + "units_per_layer": round(units, 1), "resident_gb": round(units * n_layers * UNIT_BYTES / 1e9, 1), "ssd_mb_per_tok": round(ssd / 1e6, 1), "ssd_gbs": round(ssd * tok_s / 1e9, 2), "read_gb_per_tok": round(read / 1e9, 2), "tok_s_bound": round(bandwidth_gbs * 1e9 / read, 1), } @@ -53,15 +62,59 @@ def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: return out +def session_kl(spec: str) -> dict[str, np.ndarray]: + """``EVAL_DIR:VARIANT`` -> {session: per-position KL}.""" + + d, _, name = spec.rpartition(":") + return {p.stem: np.load(p)["kl"].astype(np.float64) for p in sorted((Path(d).expanduser() / name).glob("*.npz")) + if p.stem != "admissions"} + + +def compare(a: str, b: str, reps: int = 2000, block: int = 64, seed: int = 0) -> dict: + """Paired block bootstrap of the domain-mean KL difference A - B. + + Positions are resampled in blocks of ``block`` within each session (the same blocks for A and + B), sessions are pooled by domain as in ``local_eval.py``, and domains are averaged.""" + + ka, kb = session_kl(a), session_kl(b) + da, _, na = a.rpartition(":") + doms = {s: r["domain"] for s, r in json.loads((Path(da).expanduser() / "summary.json").read_text())["variants"][na].items() + if not s.startswith("_")} + assert set(ka) == set(kb) == set(doms), "the two variants cover different sessions" + rng = np.random.default_rng(seed) + + def stat(pick) -> float: + num: dict[str, float] = {} + den: dict[str, int] = {} + for s in ka: + i = pick(len(ka[s])) + num[doms[s]] = num.get(doms[s], 0.0) + float((ka[s][i] - kb[s][i]).sum()) + den[doms[s]] = den.get(doms[s], 0) + len(i) + return float(np.mean([num[d] / den[d] for d in num])) + + def blocks(n: int) -> np.ndarray: + starts = rng.integers(0, max(n - block, 0) + 1, size=-(-n // block)) + return (starts[:, None] + np.arange(min(block, n))[None, :]).ravel()[:n] + + point = stat(lambda n: np.arange(n)) + draws = np.sort([stat(blocks) for _ in range(reps)]) + return {"a": a, "b": b, "kl_diff": round(point, 4), "ci95": [round(float(draws[int(0.025 * reps)]), 4), + round(float(draws[int(0.975 * reps)]), 4)]} + + def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("--eval", action="append", required=True, type=Path) + p.add_argument("--eval", action="append", default=[], type=Path) + p.add_argument("--compare", nargs=2, action="append", default=[], metavar=("A", "B"), + help="EVAL_DIR:VARIANT pairs: paired block-bootstrap 95%% interval of domain-mean KL(A) - KL(B)") p.add_argument("--tok-s", type=float, default=30.0) p.add_argument("--bandwidth-gbs", type=float, default=546.0) args = p.parse_args(argv) for d in args.eval: for row in rows(d.expanduser(), args.tok_s, args.bandwidth_gbs): print(json.dumps(row)) + for a, b in args.compare: + print(json.dumps(compare(a, b))) return 0 diff --git a/tests/test_runpod_v41_routed_precision.py b/tests/test_runpod_v41_routed_precision.py index 6872c6d..662f2d9 100644 --- a/tests/test_runpod_v41_routed_precision.py +++ b/tests/test_runpod_v41_routed_precision.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(REPO / "src")) import local_eval # noqa: E402 +import routed_report # noqa: E402 import precision_alloc # noqa: E402 @@ -62,3 +63,30 @@ def test_nested_decoder_matches_local_eval(): grid = e8_1bit_grid().astype(np.float32) want = (grid[z["w1_codes"][1].astype(np.int64)] * z["w1_scales"][1].astype(np.float32)[..., None]).reshape(4, -1) assert np.allclose(got, want) + + +def test_residual_cache_oracle_and_trace(): + ind = np.array([[0, 1], [2, 1], [2, 0], [3, 3]]) + wts = np.array([[0.9, 0.1], [0.5, 0.1], [0.5, 0.2], [0.3, 0.3]]) + counts = np.zeros((4, 5)) + # record 0 = tokens 0-2: expert 0 has mass 1.1, expert 2 has 1.0, expert 1 has 0.2 -> hot {0, 2} + hi = local_eval.residual_cache(ind, wts, k=2, tau=0.3, bounds=[3, 1], warm=None, counts=counts, policy="oracle") + assert hi.tolist() == [[True, False], [True, False], [True, True], [True, True]] + assert counts[:, 4].sum() == 0 # the oracle never fetches + trace: list = [] + local_eval.residual_cache(ind, wts, k=2, tau=0.3, bounds=[3, 1], warm=None, counts=counts, trace=trace) + assert trace == [(0, 0, 0), (0, 1, 2), (1, 0, 3)] # (record, token in record, expert); gate 0.2 never admits + + +def test_compare_bootstrap(tmp_path): + import json + + rng = np.random.default_rng(1) + sessions = {f"s{i}": {"domain": f"d{i % 2}"} for i in range(4)} + for name, shift in (("a", 0.1), ("b", 0.0)): + (tmp_path / name).mkdir() + for s in sessions: + np.savez(tmp_path / name / f"{s}.npz", kl=np.abs(np.random.default_rng(hash(s) % 97).normal(1, 0.2, 500)) + shift) + (tmp_path / "summary.json").write_text(json.dumps({"variants": {"a": sessions, "b": sessions}})) + out = routed_report.compare(f"{tmp_path}:a", f"{tmp_path}:b", reps=200) + assert abs(out["kl_diff"] - 0.1) < 1e-9 and out["ci95"][0] <= 0.1 <= out["ci95"][1] From ffab2bf70a5cfca391be1fd832c3b16b70329495 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:44:36 -0800 Subject: [PATCH 33/74] v41 routed precision: default code_bits 16 for the first E8P runs (separate-copy mode crashed) --- scripts/runpod/v41_reap/keep_quant.py | 3 ++- scripts/runpod/v41_reap/local_eval.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/runpod/v41_reap/keep_quant.py b/scripts/runpod/v41_reap/keep_quant.py index 3d0f265..fea1ebb 100644 --- a/scripts/runpod/v41_reap/keep_quant.py +++ b/scripts/runpod/v41_reap/keep_quant.py @@ -61,7 +61,8 @@ def decode_codes(z, proj: str, e: int) -> np.ndarray: from keep.vq.e8 import e8_1bit_grid, e8p_full_grid - grid = (e8p_full_grid() if int(z["code_bits"]) == 16 else e8_1bit_grid()).astype(np.float32) + bits = int(z["code_bits"]) if "code_bits" in z else 16 # the first E8P runs wrote no code_bits + grid = (e8p_full_grid() if bits == 16 else e8_1bit_grid()).astype(np.float32) vec = grid[z[f"{proj}_codes"][e].astype(np.int64)] # [out, in/8, 8] scales = z[f"{proj}_scales"][e].astype(np.float32) vec = vec * np.repeat(scales, vec.shape[1] // scales.shape[-1], axis=-1)[..., None] diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 2046a10..648f984 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -295,7 +295,7 @@ def tier_weights(layer: int, e: int) -> tuple[dict, dict]: if e not in cache: tiers = [] for z in cache["z"]: - grid = np_grids[int(z["code_bits"])] + grid = np_grids[int(z.get("code_bits", 16))] # the first E8P runs wrote no code_bits tier = {} for proj in ("w1", "w2", "w3"): vec = grid[z[f"{proj}_codes"][e].astype(np.int64)] From 2849338a43410f4e9b2e1703ead47bb5dfed2eb4 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:54:38 -0800 Subject: [PATCH 34/74] runpod: masked calibration (collect.py --mask), rerank.py and the X1 prune-twice pod script --- scripts/runpod/v41_reap/collect.py | 19 +++++++- scripts/runpod/v41_reap/rerank.py | 73 ++++++++++++++++++++++++++++++ scripts/runpod/v41_reap/x1_pod.sh | 50 ++++++++++++++++++++ tests/test_runpod_v41_stream.py | 28 +++++++++++- 4 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 scripts/runpod/v41_reap/rerank.py create mode 100755 scripts/runpod/v41_reap/x1_pod.sh diff --git a/scripts/runpod/v41_reap/collect.py b/scripts/runpod/v41_reap/collect.py index 052dccd..a7ab730 100644 --- a/scripts/runpod/v41_reap/collect.py +++ b/scripts/runpod/v41_reap/collect.py @@ -407,6 +407,7 @@ def run_chunk( students: dict[str, dict[int, Any]], dims: dict[str, int], device: Any, + teacher_mask: dict[int, Any] | None = None, ) -> dict[tuple[str, str], dict[str, np.ndarray]]: """One pass over the weights for ``chunk``; returns arrays per (stream, session). @@ -414,6 +415,9 @@ def run_chunk( Eval: the teacher (REAP + routes + top-K logits) and every student (scored against the teacher) side by side. Teacher records go through the MoE per session, so the stats stay per session; each student's records go through it together. + + ``teacher_mask`` (calibration only) masks the measured stream's router to a survivors table, + so the stats describe that pruned model's own routing and activations. """ import torch @@ -441,7 +445,7 @@ def before_moe(layer: int, block: Any, key: tuple[str, str | None]) -> None: if not hasattr(block, "full_bias"): block.full_bias = bias.data.clone() bias.data.copy_(block.full_bias) - mask = students[stream].get(layer) if stream != TEACHER else None + mask = students[stream].get(layer) if stream != TEACHER else (teacher_mask or {}).get(layer) if mask is not None: bias.data[mask.to(bias.device)] = float("-inf") state["stats"] = stats[name] if stream == TEACHER and layer < n_layers else None @@ -496,6 +500,11 @@ def main(argv: list[str] | None = None) -> int: help="eval sweep (repeatable): also run a student with the router masked to this reap-select " "table, scored against the teacher, into /student-KEEP/", ) + parser.add_argument( + "--mask", type=Path, + help="calibration sweep: mask the router to this survivors table, so REAP, routes and imatrix " + "are measured on that pruned model's own routing (for re-ranking after a prune)", + ) parser.add_argument("--chunk-tokens", type=int, default=524288, help="resident tokens per weight pass (x streams)") parser.add_argument("--shard", default="0/1", help="I/N: this process takes every Nth session from I") parser.add_argument("--max-seq-len", type=int, default=16384) @@ -506,6 +515,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.survivors and args.sweep != "eval": parser.error("--survivors needs --sweep eval") + if args.mask and args.sweep != "calibration": + parser.error("--mask needs --sweep calibration") shard, n_shards = map(int, args.shard.split("/")) import torch @@ -546,6 +557,7 @@ def main(argv: list[str] | None = None) -> int: students[keep] = survivor_masks(json.loads(Path(table).read_text()), n_experts) survivors_sha[keep] = hashlib.sha256(Path(table).read_bytes()).hexdigest() + teacher_mask = survivor_masks(json.loads(args.mask.read_text()), n_experts) if args.mask else None run_config = { "model_id": MODEL_ID, "revision": PINNED_REVISION, @@ -555,6 +567,8 @@ def main(argv: list[str] | None = None) -> int: "top_k": args.top_k, "driver": "layer-streamed, world_size 1", } + if args.mask: + run_config["mask_sha256"] = hashlib.sha256(args.mask.read_bytes()).hexdigest() text = json.dumps(run_config, indent=2, sort_keys=True) + "\n" config_path = args.out / "run-config.json" args.out.mkdir(parents=True, exist_ok=True) @@ -587,7 +601,8 @@ def target(stream_name: str, name: str) -> Path: print("deadline reached; stopping cleanly", flush=True) break t0 = time.time() - results = run_chunk(streamer, state, chunk, split=args.sweep, students=students, dims=dims, device="cuda") + results = run_chunk(streamer, state, chunk, split=args.sweep, students=students, dims=dims, device="cuda", + teacher_mask=teacher_mask) for name, records in chunk: ident = identity(session=name, split=args.sweep, records=records, config_sha=config_sha, chunk=args.max_seq_len) for stream_name in streams: diff --git a/scripts/runpod/v41_reap/rerank.py b/scripts/runpod/v41_reap/rerank.py new file mode 100644 index 0000000..71100c8 --- /dev/null +++ b/scripts/runpod/v41_reap/rerank.py @@ -0,0 +1,73 @@ +"""Re-rank experts by total routed mass (sum of g * ||f||), optionally inside an earlier survivors set. + +"Prune twice": measure routing on the model that is already pruned (``collect.py --sweep +calibration --mask TABLE``), then keep each layer's top K of that table by the new mass. On the +Mac this cut KL by 0.18 going from 192 to 96 experts; neither the mass criterion nor the +pruned-model routing did so alone (recovery log, step 2c). + + python rerank.py --stats OUT/calibration/sessions --within fm-288.json --keep 192 --out pt-192.json + +``--stats`` is a directory of ``collect.py`` session files or one npz holding +``reap__gate_norm_sum`` [layers, experts]. Without ``--within`` every expert is a candidate. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + + +def total_mass(stats: Path) -> np.ndarray: + paths = sorted(stats.glob("*.npz")) if stats.is_dir() else [stats] + assert paths, f"no npz under {stats}" + total = 0 + for path in paths: + with np.load(path) as z: + total = total + z["reap__gate_norm_sum"].astype(np.float64) + return total + + +def rerank(mass: np.ndarray, keep: int, within: dict[str, list[int]] | None) -> dict[str, list[int]]: + out = {} + for layer in range(mass.shape[0]): + ids = np.asarray(sorted(within[str(layer)]) if within else range(mass.shape[1])) + assert keep <= len(ids), f"layer {layer}: keep {keep} of {len(ids)}" + order = np.argsort(-mass[layer, ids], kind="stable")[:keep] + out[str(layer)] = sorted(int(e) for e in ids[order]) + return out + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--stats", required=True, type=Path) + p.add_argument("--within", type=Path, help="survivors JSON: candidates are this table's experts") + p.add_argument("--keep", required=True, type=int) + p.add_argument("--out", required=True, type=Path) + args = p.parse_args(argv) + within = json.loads(args.within.read_text())["survivors"] if args.within else None + mass = total_mass(args.stats) + table = rerank(mass, args.keep, within) + kept = np.mean([mass[int(l), ids].sum() / mass[int(l)].sum() for l, ids in table.items()]) + args.out.write_text(json.dumps({"record_type": "keep_reap_survivors_v1", "criterion": "total routed mass", + "stats": str(args.stats), "within": str(args.within) if args.within else None, + "survivors": table})) + print(json.dumps({"keep": args.keep, "layers": len(table), "mass_kept_mean": round(float(kept), 4)})) + return 0 + + +def _check() -> None: + mass = np.array([[5.0, 1.0, 4.0, 3.0], [0.0, 2.0, 9.0, 1.0]]) + assert rerank(mass, 2, None) == {"0": [0, 2], "1": [1, 2]} + assert rerank(mass, 1, {"0": [1, 3], "1": [0, 3]}) == {"0": [3], "1": [3]} # only inside the table + + +if __name__ == "__main__": + if sys.argv[1:] == ["--check"]: + _check() + print("ok") + sys.exit(0) + sys.exit(main()) diff --git a/scripts/runpod/v41_reap/x1_pod.sh b/scripts/runpod/v41_reap/x1_pod.sh new file mode 100755 index 0000000..80368d4 --- /dev/null +++ b/scripts/runpod/v41_reap/x1_pod.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# X1 on one GPU pod: "prune twice" at the first cut (384 -> 288 -> 192 -> 96), scored against the +# shipped REAP tables. Plan: docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md, section 7. +# +# The volume already holds everything (a CPU pod put it there, so the GPU starts work at once): +# /workspace/v41-hf the pinned checkpoint, fetch verified +# /workspace/pack the calibration + eval pack +# /workspace/wheels pip wheels for inference/requirements.txt +# /workspace/x1/*.json fm-288, fm-192 (total mass on full-model routing), survivors-192 (shipped +# REAP-50), pm-96 (the Mac's pick inside REAP-50) +set -euo pipefail +W=/workspace +KIT=$W/keep/scripts/runpod/v41_reap +HF=$W/v41-hf +X=$W/x1 +CAL_SHARD=${CAL_SHARD:-0/12} # 2 of 24 calibration sessions, about 256K tokens, every domain +EVAL_SHARD=${EVAL_SHARD:-0/2} # 54 of 108 eval sessions +CHUNK=${CHUNK_TOKENS:-262144} +export PYTHONPATH=$W/keep/src MAX_GPU_HOURS=${MAX_GPU_HOURS:-9} +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$X/x1.log"; } +trap 'status=$?; [[ $status -ne 0 ]] && { log "FAILED (status $status)"; touch "$X/FAILED"; }' EXIT + +pip install -q --no-index --find-links "$W/wheels" -r "$KIT/requirements-ref.txt" \ + || pip install -q -r "$KIT/requirements-ref.txt" +pip freeze > "$W/pip-freeze-gpu.txt" # pod.sh's gpu_setup then skips its own install +nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$X/x1.log" + +log "smoke" +bash "$KIT/pod.sh" smoke + +calibrate() { # name, mask + log "calibration masked to $2 ($CAL_SHARD)" + python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/pack/v41-pack.jsonl" --out "$X/$1" --sweep calibration \ + --mask "$2" --shard "$CAL_SHARD" --chunk-tokens "$CHUNK" > "$X/cal-$1.log" 2>&1 + log "calibration $1 done: $(tail -1 "$X/cal-$1.log")" +} +calibrate m288 "$X/fm-288.json" +python3 "$KIT/rerank.py" --stats "$X/m288/calibration/sessions" --within "$X/fm-288.json" --keep 192 --out "$X/pt-192.json" | tee -a "$X/x1.log" +calibrate m192 "$X/pt-192.json" +python3 "$KIT/rerank.py" --stats "$X/m192/calibration/sessions" --within "$X/pt-192.json" --keep 96 --out "$X/pt-96.json" | tee -a "$X/x1.log" + +log "eval ($EVAL_SHARD): teacher + reap192, fm192, pt192, pm96, pt96" +python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/pack/v41-pack.jsonl" --out "$X/eval" --sweep eval \ + --shard "$EVAL_SHARD" --chunk-tokens "$CHUNK" \ + --survivors reap192="$X/survivors-192.json" --survivors fm192="$X/fm-192.json" --survivors pt192="$X/pt-192.json" \ + --survivors pm96="$X/pm-96.json" --survivors pt96="$X/pt-96.json" > "$X/eval.log" 2>&1 +python3 "$KIT/score.py" --teacher "$X/eval" --student reap192="$X/eval/student-reap192" --student fm192="$X/eval/student-fm192" \ + --student pt192="$X/eval/student-pt192" --student pm96="$X/eval/student-pm96" --student pt96="$X/eval/student-pt96" > "$X/scores.json" +log "scores written" +touch "$X/DONE" diff --git a/tests/test_runpod_v41_stream.py b/tests/test_runpod_v41_stream.py index 742f713..d015e3b 100644 --- a/tests/test_runpod_v41_stream.py +++ b/tests/test_runpod_v41_stream.py @@ -183,11 +183,12 @@ def _reference_stats(toy, collect, records, dims): return stats.arrays(LAYERS), hidden -def _sweep(toy, collect, monkeypatch, chunk, split, students=None): +def _sweep(toy, collect, monkeypatch, chunk, split, students=None, teacher_mask=None): streamer = _streamer(toy) monkeypatch.setattr(toy.ref.MoE, "forward", toy.ref.MoE.forward) # install_hooks patches it; undo at teardown state = collect.install_hooks(streamer, top_k=16, rows_per_slice=7) - return collect.run_chunk(streamer, state, chunk, split=split, students=students or {}, dims=DIMS, device="cpu"), streamer + return collect.run_chunk(streamer, state, chunk, split=split, students=students or {}, dims=DIMS, device="cpu", + teacher_mask=teacher_mask), streamer def _assert_stats_match(got, want): @@ -213,6 +214,29 @@ def test_calibration_sweep_matches_reference(toy, monkeypatch): assert results[("teacher", "cal-0")]["routes__expert_ids"].shape == (60, 6, 2) +def test_masked_calibration_measures_the_pruned_models_routing(toy, monkeypatch): + import collect + + chunk = [("cal-0", [_record(toy.records[0], "cal-0", "calibration")])] + gen = torch.Generator().manual_seed(3) + kept = {str(layer): sorted(torch.randperm(8, generator=gen)[:5].tolist()) for layer in LAYERS} + mask = collect.survivor_masks({"survivors": kept}, 8) + with torch.no_grad(): + full = {layer: toy.model.layers[layer].ffn.gate.bias.clone() for layer in LAYERS} + for layer, m in mask.items(): + toy.model.layers[layer].ffn.gate.bias[m] = float("-inf") + want = _reference_stats(toy, collect, chunk[0][1], DIMS)[0] + for layer, bias in full.items(): + toy.model.layers[layer].ffn.gate.bias.copy_(bias) + results, _ = _sweep(toy, collect, monkeypatch, chunk, "calibration", teacher_mask=mask) + got = results[("teacher", "cal-0")] + _assert_stats_match(got, want) + for layer in LAYERS: # nothing routes to a masked expert, and the unmasked run differs + assert got["reap__route_count"][layer][mask[layer].numpy()].sum() == 0 + plain, _ = _sweep(toy, collect, monkeypatch, chunk, "calibration") + assert not np.array_equal(plain[("teacher", "cal-0")]["reap__route_count"], got["reap__route_count"]) + + def test_eval_sweep_teacher_and_students_match_reference(toy, monkeypatch): import collect From fd3d57ed9d9b2bc38626e2359d9ce05555e983e1 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:55:00 -0800 Subject: [PATCH 35/74] docs: X2 closed (format artifact), X1 pod run started --- .../2026-11-08-v41-reap-recovery-log.md | 16 ++++++++++++++++ .../2026-11-09-keep-findings-and-levers.md | 16 +++++++++++++--- .../2026-11-09-routed-precision-plan.md | 2 +- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index d04d18c..3b7e5e7 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -633,3 +633,19 @@ to 100; partial credit only for measured progress). projection-level rungs (`--hi-proj`), admission traces (`--trace`), expert masks under `--dynamic`, and `routed_report.py --compare`. Passes 2, 2b and 3 are queued behind pass 1 (`~/Models/v41-reap-run/jobs/queue2.sh`, 24 variants, about 11 h). + +### 2026-11-09: X1 on a pod (prune twice at the first cut), and X2 closed + +- X2 (the pruned model's better tool-use NLL) is an artifact, settled from the pod's stored + outputs: the advantage exists only on positions where the teacher is surprised by off-format text + inside the records. Details in the findings note. +- X1 started. Design: mask the full model's router to the top 288 experts by total routed mass + (`x1/fm-288.json`, 97.5% of mass), run calibration on that masked model (`collect.py --mask`, 2 of + 24 sessions, about 256K tokens), keep the top 192 by the new mass (`pt-192`), repeat once to 96 + (`pt-96`), then score five students against the teacher on 54 eval sessions: shipped REAP-50, + one-pass total mass at 192 (`fm-192`, the criterion alone; it overlaps REAP-50 by 76.5%), + `pt-192`, the Mac's 96 inside REAP-50 (`pm-96`), and `pt-96`. Script: `x1_pod.sh`. +- Runpod, US-NE-1 (H100 SXM Low there, RTX PRO 6000 as fallback, CPU pods available): network volume + `yqazs7aqe6` (600 GB) and CPU pod `hd0czk7klgonpi` (4 vCPU, $0.12/h) for the 510 GB fetch, the + wheels and the inputs, so the GPU pod starts on GPU work. Estimated GPU time 6.5–7.5 h at + $3.99/h. The volume stays for W4b. diff --git a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md index 58adfb6..76a3359 100644 --- a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md +++ b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md @@ -138,9 +138,19 @@ In the order I would bet on them. X numbers refer to the plan. model loses precision instead of swapping. KEEP has already measured the failure this avoids (535,792 swapouts, 16.5× timing spread at 91 GB). This is specific to how unified memory and the macOS VM behave, and it turns "SSD churn" from writes into reads. -4. **Pruning as specialization (X2).** If the tool-use result holds up against the resident full - model and on a downstream benchmark, a calibration-targeted prune is not only smaller but better - on its target domains. Large if true; the prior is that it is partly a teacher artifact. +4. **Pruning as specialization (X2): checked the same day, and it is not that.** On the pod's + full-length outputs (27 eval sessions held locally) the student-192 NLL advantage appears in + every agent-style domain (tool-use −1.38 nats/token, agentic-python −0.33, agentic-rust −0.32, + bash −0.16), but only where the teacher is surprised. On positions where the teacher's NLL is at + most 2 the student is worse in every domain (+0.14 on tool-use). On tool-use, positions with + teacher NLL above 5 are 30% of the tokens and carry −1.29 of the −1.38. At the worst of them the + teacher's top prediction is its own tool-call markup (`｜DSML｜` after `<`) or end-of-sentence + after `>`, while the record continues with foreign markup that the source datasets embed in + message text. The full model is confidently wrong about text that is off its own format; the + pruned model is less confident and is penalized less. That is a softer distribution on + off-format data, not a better model. Two consequences: NLL on these domains is not a quality + measure, and KL against the teacher there mostly measures agreement on format expectations. + Lever dropped; the eval caveat stays. 5. **One artifact for every budget.** Because quantization statistics survive pruning and one mass ranking yields nested expert sets (pm-144 ⊂ pm-160; 97% overlap between passes), a single set of files can serve any memory budget: nested in expert count and in bits. MatMoE nests bits only. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 7e89e22..dc57509 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -253,7 +253,7 @@ literature yet; each gets a search before it is claimed. Ordered by expected val | # | Lever | Evidence in hand | Test | Where it runs | |---|---|---|---|---| | X1 | Prune twice at 384 → 192. REAP-50 was chosen on full-model routing with the per-route score; its KL (0.578) is the floor under everything here. | The same fix cut KL 15% at 192 → 96 (step 2c ablation); a second pass saturates (step 2d). | Calibration pass on the full model masked to 288 by total mass, re-rank to 192, eval against shipped REAP-50. | Pod, with W4b; about $15–25 extra | -| X2 | Calibration-targeted pruning as specialization. | REAP-50 beats the full model by 1.22 nats/token on tool-use NLL; the pod's own student shows the same gap. Logged as an anomaly, never chased. | Per-position NLL of teacher and students on tool-use with the full model resident; then a downstream tool-use benchmark (W8). | Pod, same run as X1 | +| X2 | ~~Calibration-targeted pruning as specialization.~~ Dropped 2026-11-09. | The NLL advantage sits only where the teacher is surprised by off-format text embedded in the records; on teacher-confident positions the student is worse in every domain. | Done locally from the pod's stored outputs; see the findings note, section 3. | — | | X3 | Keep every expert, offload residuals, instead of pruning half. | V4.1 plan: each expert kept costs 0.359 GB of RAM at 2 bits "but adds no per-token reads", and the fit "forces a ~50% prune". All 384 one-bit bases cost the same memory as REAP-50 at 2 bits. | 384 bases + K residuals against REAP-50 at 2 bits (0.747) at equal resident memory. This is the title's claim at full scale. | Pod, W4b; the Mac sweep at 192 vs 96 is its proxy | | X4 | The OS page cache as the precision cache. Residuals are memory-mapped clean file pages; `mincore` asks whether one is resident, `madvise(WILLNEED)` requests it in the background, and the kernel evicts under pressure at no write cost. Precision degrades instead of the machine swapping. | Roadmap, 2026-08-21: at 90.99 GB of MLX-active memory byte-identical work spanned 16.5× in time and the bind alone logged 535,792 swapouts. Anonymous memory at that footprint thrashes; clean file pages cannot. | In W6: replay the trace through `mmap` + `mincore`, with and without a memory hog, and measure hits, latency and swapouts. | Mac | | X5 | A stronger base and finer rungs. The 1-bit base is the weak link. | Pilot: 1-bit codes gave a 0.56 block cosine; the SwiGLU amplifies projection error about 2.8×; 2 bits is "at its rate-distortion limit". Here, a static 1-bit tier added 0.025 KL at best. | (a) Projection-level tiers from the existing codes: residual for the down projection only (1.33 bits) or gate/up only (1.67 bits), no new fit. (b) The 2-bit base + 1-bit residual ladder, moved up from fallback to W4 pass 3. | Mac | From 35bff846ebaac172a608d18ac5979ab591b79098 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:41:53 -0800 Subject: [PATCH 36/74] v41 sidecars: unmap Engram tables after use, resumable fit, partial-fit eval; lever closed (+0.199 KL at 15 layers) --- .../2026-11-08-v41-reap-recovery-log.md | 24 +++++++++++ scripts/runpod/v41_reap/local_eval.py | 4 ++ scripts/runpod/v41_reap/sidecar_fit.py | 40 ++++++++++++++----- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 3b7e5e7..b3cb763 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -649,3 +649,27 @@ to 100; partial credit only for measured progress). `yqazs7aqe6` (600 GB) and CPU pod `hd0czk7klgonpi` (4 vCPU, $0.12/h) for the 510 GB fetch, the wheels and the inputs, so the GPU pod starts on GPU work. Estimated GPU time 6.5–7.5 h at $3.99/h. The volume stays for W4b. + +### 2026-11-09: step-3 sidecars closed (worse end to end), and an eval-set caveat + +- The 4,096-token sidecar fit (rank 256, 62 records balanced over 13 domains, about 150K tokens) + stopped at layer 15 of 40: peak RSS 81.8 GB passed the 80 GB guard right after layer 14. Layers + 1 and 14 are the two Engram layers; the pages their lookups touch stayed mapped. `sidecar_fit.py` + now unmaps each table after its layer and resumes from saved layers (resume not yet exercised). +- Before resuming, the 15 fitted layers (0–14) were scored alone (`evals/step3-sidecar-l15`; + `local_eval.py --sidecar` now accepts a partial fit). Against the same 96 experts at 2 bits + without sidecars: KL 1.238 against 1.039, top-1 0.705 against 0.733; paired bootstrap + **+0.199 [+0.164, +0.219]**. The median per-position change is 0.0006. The harm is in + english-prose (+0.71, +1.15), english-chat (+0.34, +0.59) and business (+0.37, +0.48); the code, + agentic and tool-use sessions move by −0.11 to +0.12. +- Layer-local holdout numbers said the opposite (holdout residual energy down 8–19% in the + large-gap layers, 79–93% in layers 8, 10 and 14). Cause not established. Energy-weighted error + on the fit's own records is not KL; an application mismatch between fit and eval is not excluded, + though it would be expected to hurt every domain. Same shape as the GLM-4.5-Air sidecar: a proxy + win that failed end to end. The fit is not resumed and the lever is closed. +- Eval-set caveat found on the way: four of the 22 Mac eval sessions (`eval-00000`, `-00001`, + `-00004`, `-00005`; both agentic-python and both agentic-rust) have identical first 1,024 + tokens, a shared system prompt. At 1,024 tokens the Mac set is 19 distinct texts and those two + domains measure the same one. Paired differences between variants are unaffected; domain means + at 1,024 tokens overweight that prompt. The 4,096-token confirmation pass (W4 pass 4) is where + those sessions diverge. diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 648f984..be4f243 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -349,6 +349,8 @@ def dynamic_forward(self, x, image_mask=None): if args.sidecar: # sidecar_fit.py output: y += ([x, 1] @ U) @ Vt on every MoE call sidecars = {} for layer in range(int(model_args.n_layers)): + if not (args.sidecar / f"layer-{layer:02d}.npz").exists(): + continue # a fit still in progress: the layers it has are valid without the rest with np.load(args.sidecar / f"layer-{layer:02d}.npz") as z: U, Vt = z["U"][:, : args.sidecar_rank or None], z["Vt"][: args.sidecar_rank or None] sidecars[layer] = (torch.as_tensor(U), torch.as_tensor(Vt)) @@ -356,6 +358,8 @@ def dynamic_forward(self, x, image_mask=None): def with_sidecar(self, x, image_mask=None): y = moe_forward(self, x, image_mask) + if self.layer_id not in sidecars: + return y U, Vt = sidecars[self.layer_id] xf = x.reshape(-1, x.size(-1)).float() corr = (xf @ U[:-1] + U[-1]) @ Vt diff --git a/scripts/runpod/v41_reap/sidecar_fit.py b/scripts/runpod/v41_reap/sidecar_fit.py index 14b06f9..2c8b615 100644 --- a/scripts/runpod/v41_reap/sidecar_fit.py +++ b/scripts/runpod/v41_reap/sidecar_fit.py @@ -169,6 +169,9 @@ def load_with_codes(prefix, skip=()): # prefetch thread: the next layer's stude streamer.load_layer = load_with_codes args.out.mkdir(parents=True, exist_ok=True) + # Resume: a layer with a metrics row keeps its saved sidecar; only the student is run through it. + done = {json.loads(line)["layer"] for line in (args.out / "metrics.jsonl").read_text().splitlines()} \ + if (args.out / "metrics.jsonl").exists() else set() metrics = (args.out / "metrics.jsonl").open("a") def expert(x, w, w1, w2, w3, limit): @@ -182,12 +185,23 @@ def moe_forward(self, x, image_mask=None): shape, layer = x.size(), self.layer_id x = x.view(-1, dim) local = ids[layer] + # This layer's Engram lookups are done. Unmapping the table drops the pages they touched + # (tens of GB at 150k tokens) from RSS; left mapped, layers 1 and 14 together passed 80 GB. + engram_rows.pop(layer, None) + n = x.size(0) + + def blocks(): # [x, 1] in row blocks, so no full fp32 copy of x is ever resident + for s in range(0, n, args.block_rows): + e = min(n, s + args.block_rows) + yield s, e, torch.cat([x[s:e].float(), torch.ones(e - s, 1, device=x.device)], dim=1) + full_bias = self.gate.bias.data.clone() - weights, indices = self.gate(x, None) # teacher: every expert in the checkpoint - y_t = torch.zeros_like(x, dtype=torch.float32) - for i in torch.unique(indices).tolist(): - idx, top = torch.where(indices == i) - y_t[idx] += self.experts[i](x[idx], weights[idx, top, None]).float() + if layer not in done: + weights, indices = self.gate(x, None) # teacher: every expert in the checkpoint + y_t = torch.zeros_like(x, dtype=torch.float32) + for i in torch.unique(indices).tolist(): + idx, top = torch.where(indices == i) + y_t[idx] += self.experts[i](x[idx], weights[idx, top, None]).float() keep = set(student[layer]) self.gate.bias.data[torch.as_tensor([e not in keep for e in local], device=x.device)] = float("-inf") weights, indices = self.gate(x, None) # student: masked router, KEEP-decoded experts @@ -200,18 +214,22 @@ def moe_forward(self, x, image_mask=None): w = {p: t.to(x.device) for p, t in weights_of[local[i]].items()} y_s[idx] += expert(x[idx], weights[idx, top, None], w["w1"], w["w2"], w["w3"], limit).float() + if layer in done: + if args.apply: + with np.load(args.out / f"layer-{layer:02d}.npz") as z: + U, Vt = torch.as_tensor(z["U"], device=x.device), torch.as_tensor(z["Vt"], device=x.device) + for s, e, xt in blocks(): + y_s[s:e] += (xt @ U) @ Vt + print(json.dumps({"layer": layer, "resumed": True, "seconds": round(time.time() - t0, 1)}), flush=True) + y_s += self.shared_experts(x).float() + return y_s.type_as(x).view(shape) + energy = {s: float(y_t[m].pow(2).sum()) for s, m in (("fit", ~holdout), ("holdout", holdout))} r = y_t.sub_(y_s) # in place: y_t is not needed past its energy del y_t - n = x.size(0) A = torch.zeros(dim + 1, dim + 1, dtype=torch.float64, device=x.device) B = torch.zeros(dim + 1, dim, dtype=torch.float64, device=x.device) - def blocks(): # [x, 1] in row blocks, so no full fp32 copy of x is ever resident - for s in range(0, n, args.block_rows): - e = min(n, s + args.block_rows) - yield s, e, torch.cat([x[s:e].float(), torch.ones(e - s, 1, device=x.device)], dim=1) - for s, e, xt in blocks(): m = ~holdout[s:e] A += (xt[m].T @ xt[m]).double() From 51c0eddc0f5e9893b5fb2da5c5a26870c4333b16 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:33:26 -0800 Subject: [PATCH 37/74] runpod x1: copy the checkpoint to the container disk before GPU work (network volume is 120 MB/s single-stream); log --- .../2026-11-08-v41-reap-recovery-log.md | 20 +++++++++++++++++++ scripts/runpod/v41_reap/pod.sh | 2 +- scripts/runpod/v41_reap/x1_pod.sh | 16 ++++++++++++++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index b3cb763..04b8dda 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -673,3 +673,23 @@ to 100; partial credit only for measured progress). domains measure the same one. Paired differences between variants are unaffected; domain means at 1,024 tokens overweight that prompt. The 4,096-token confirmation pass (W4 pass 4) is where those sessions diverge. + +### 2026-11-09: X1 running on an H100, after 70 lost minutes + +- GPU pod `q8vcl6ob0hf0lk` (H100 SXM, US-NE-1, $3.99/h) created 01:01 UTC once the fetch verified; + the CPU pod was terminated. Calibration on the 288-expert masked model started 02:31 UTC. The 90 + minutes between cost about $6 and did no GPU work: + - 15 minutes: my readiness probe compared the proxy's reply to a prefix, and the reply starts + with a terminal escape sequence, so it waited out all its tries on a pod that was already up. + - One failed launch: the Ubuntu 24.04 image marks the system Python as externally managed + (`PIP_BREAK_SYSTEM_PACKAGES=1` in `x1_pod.sh`). + - 50 minutes: the smoke test sat at 0% GPU. The network volume reads one stream at about + 120 MB/s and a random 256-byte row in 2–3 ms; the loader reads shards through one mmap and + the Engram lookup faults in rows one page at a time. Sixteen parallel streams read 3.2 GB/s + across files and 1.1 GB/s inside one file. +- Fix: container disk raised from 50 to 560 GB (restarts the container), and `x1_pod.sh` copies + the checkpoint to the container's own disk with 16 parallel `cp` and checks the byte total + (`LOCAL_HF=/root/v41-hf`; `pod.sh` and `x1_pod.sh` take `HF` from the environment). The copy took + 7 minutes for 476 GB. Smoke then passed: 16,384 tokens in 447 s, 48.1 GB peak GPU memory. +- For W4b and any later pod on this volume: budget the 7-minute copy, create the pod with a + 560 GB container disk from the start, and probe readiness by substring. diff --git a/scripts/runpod/v41_reap/pod.sh b/scripts/runpod/v41_reap/pod.sh index 1d5e605..2a2b432 100755 --- a/scripts/runpod/v41_reap/pod.sh +++ b/scripts/runpod/v41_reap/pod.sh @@ -18,7 +18,7 @@ MODEL_ID=deepseek-ai/DeepSeek-V4.1-Flash-1017 REVISION=1bae0e1088993b9b82de08f59de5be5a34e42170 W=${WORKSPACE:-/workspace} KIT=$W/keep/scripts/runpod/v41_reap -HF=$W/v41-hf +HF=${HF:-$W/v41-hf} OUT=$W/out CANDIDATES=${CANDIDATES:-"256 192 128 96"} PRUNE=${PRUNE:-"192"} # REAP 50%. REAP 75% (96) after the Mac pulls 192: 1.2 TB holds one diff --git a/scripts/runpod/v41_reap/x1_pod.sh b/scripts/runpod/v41_reap/x1_pod.sh index 80368d4..de7617e 100755 --- a/scripts/runpod/v41_reap/x1_pod.sh +++ b/scripts/runpod/v41_reap/x1_pod.sh @@ -11,15 +11,29 @@ set -euo pipefail W=/workspace KIT=$W/keep/scripts/runpod/v41_reap -HF=$W/v41-hf +HF=${HF:-$W/v41-hf} X=$W/x1 CAL_SHARD=${CAL_SHARD:-0/12} # 2 of 24 calibration sessions, about 256K tokens, every domain EVAL_SHARD=${EVAL_SHARD:-0/2} # 54 of 108 eval sessions CHUNK=${CHUNK_TOKENS:-262144} export PYTHONPATH=$W/keep/src MAX_GPU_HOURS=${MAX_GPU_HOURS:-9} +export PIP_BREAK_SYSTEM_PACKAGES=1 # Ubuntu 24.04 images mark the system Python as externally managed log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$X/x1.log"; } trap 'status=$?; [[ $status -ne 0 ]] && { log "FAILED (status $status)"; touch "$X/FAILED"; }' EXIT +# The network volume reads one stream at about 120 MB/s and a random Engram row in about 3 ms, so the +# smoke test sat at 0% GPU for 50 minutes. Sixteen parallel streams read 3 GB/s: copy the checkpoint +# to the container's own disk first (LOCAL_HF, needs ~480 GB there) and run from that. +if [[ -n "${LOCAL_HF:-}" ]]; then + log "copy checkpoint to $LOCAL_HF" + mkdir -p "$LOCAL_HF" + (cd "$W/v41-hf" && find . -type f -not -path './.cache/*' -print0 | xargs -0 -P 16 -I{} cp --parents {} "$LOCAL_HF/") + [[ $(cd "$W/v41-hf" && find . -type f -not -path './.cache/*' -printf '%s\n' | awk '{s+=$1} END {print s}') == $(cd "$LOCAL_HF" && find . -type f -printf '%s\n' | awk '{s+=$1} END {print s}') ]] + HF=$LOCAL_HF + log "copy done" +fi +export HF + pip install -q --no-index --find-links "$W/wheels" -r "$KIT/requirements-ref.txt" \ || pip install -q -r "$KIT/requirements-ref.txt" pip freeze > "$W/pip-freeze-gpu.txt" # pod.sh's gpu_setup then skips its own install From 849b442e41227901f2d12a375a3aa5e6438e06d4 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:32:58 -0800 Subject: [PATCH 38/74] docs: routed sweep pass 1 (R beats S by 0.068, loses to flat 144 by 0.025), nested fit complete, rubric 31 --- .../2026-11-08-v41-reap-recovery-log.md | 56 ++++++++++++++++-- docs/research/2607.05147v1.pdf | Bin 0 -> 1055890 bytes 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 docs/research/2607.05147v1.pdf diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 04b8dda..75dc621 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -595,16 +595,16 @@ to 100; partial credit only for measured progress). | # | Item | Weight | Score | State (2026-11-09) | |---|---|---:|---:|---| -| 1 | Nested codes, all layers, validated | 10 | 5 | fit 6/40 layers; +12% error vs flat 2-bit | +| 1 | Nested codes, all layers, validated | 10 | 10 | fit 40/40; error 0.102 gate/up, 0.084 down; end to end +0.053 KL against flat 2-bit | | 2 | Per-token routed tiers, implemented and tested | 10 | 10 | `--dynamic/--tau/--tau-drop`, smoke-tested | | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | -| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 0 | sweep chained after the nested fit | -| 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | code exists, unrun | +| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 0 | pass 1: R beats S by 0.068, but loses to 144 flat 2-bit experts by 0.025; D pending (pass 2b) | +| 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | | 6 | Real Apple Silicon streaming measured, not simulated | 10 | 0 | | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **26** | | +| | **Total** | 100 | **31** | | ### 2026-11-09: prior art, plan, restart @@ -693,3 +693,51 @@ to 100; partial credit only for measured progress). 7 minutes for 476 GB. Smoke then passed: 16,384 tokens in 447 s, 48.1 GB peak GPU memory. - For W4b and any later pod on this volume: budget the 7-minute copy, create the pod with a 560 GB container disk from the start, and probe readiness by substring. + +### 2026-11-09: routed sweep, pass 1 (192 bases, 22 sessions, first 1,024 tokens) + +Nested fit finished: 40 of 40 layers, total error 0.102 (gate, up) and 0.084 (down) against 0.090 +for the flat 2-bit fit. Sweep: 12,415 s forward, peak RSS 64.1 GB. Units are 1-bit experts per +layer; SSD rate is at 30 tok/s. + +| Variant | Units | KL | Top-1 | Tier-2 slots | SSD GB/s | +|---|---:|---:|---:|---:|---:| +| all2: every residual resident, all slots tier 2 | 384 | 0.800 | 0.779 | 100% | 0 | +| t22 (M): all resident, tier 2 at gate ≥ 0.22 | 384 | 0.822 | 0.767 | 46% | 0 | +| d10: t22 plus tier 0 below gate 0.1 | 384 | 0.823 | 0.763 | 46% (7.7% dropped) | 0 | +| c96t2 (R): LRU of 96 residuals, τ 0.2 | 288 | 0.832 | 0.772 | 90% | 0.83 | +| c96t3 (R): LRU of 96, τ 0.3 | 288 | 0.837 | 0.771 | 84% | 0.28 | +| s96 (S): static hot set of 96 | 288 | 0.905 | 0.753 | 75% | 0 | +| c48t2 (R): LRU of 48, τ 0.2 | 240 | 0.876 | 0.758 | 76% | 2.72 | +| all1: base only | 192 | 1.358 | 0.637 | 0% | 0 | + +Static references from step 2, same sessions: 192 experts at flat 2 bits 0.747 (384 units); +144 experts at flat 2 bits 0.812 (288 units); 96 at flat 2 bits 1.039 (192 units). + +Paired block bootstrap (`routed_report.py --compare`): + +| A against B | KL(A) − KL(B) | 95% interval | +|---|---:|---| +| R c96t3 against S s96, both 288 units | −0.068 | −0.077 to −0.060 | +| R c96t2 against S s96 | −0.073 | −0.084 to −0.065 | +| R c48t2 (240 units) against S s96 (288 units) | −0.029 | −0.042 to −0.022 | +| R c96t3 against 144 flat 2-bit experts, both 288 units | **+0.025** | +0.009 to +0.051 | +| R c96t2 against 144 flat 2-bit experts | +0.020 | +0.003 to +0.045 | +| all2 against 192 flat 2-bit experts (the nesting penalty) | +0.053 | +0.039 to +0.071 | +| R c96t2 (288 units) against all2 (384 units) | +0.032 | +0.023 to +0.037 | +| t22 against all2 | +0.022 | +0.015 to +0.031 | +| d10 against t22 (tier 0 on 7.7% of slots) | +0.001 | −0.002 to +0.004 | + +What it says: + +- Routing the cache works. With the same codes and the same memory, an LRU with a gate threshold + beats a fixed hot set by 0.07 KL, and a cache of 48 beats a fixed set of 96. At τ 0.3 the cost + is 9.5 MB per token, 0.28 GB/s, a ninth of the 2.5 GB/s ceiling. +- It does not yet beat plain pruning. At 288 units, 144 experts at flat 2 bits score 0.812 and the + routed cache 0.832–0.837. The nesting penalty (+0.053) is larger than that gap: with a flat-quality + high tier the routed cache would be ahead. This is the argument for a separate-copy high tier, + which pass 2b measures, and for the 2-bit-base ladder (X5). +- Tier 0 is free: dropping the 7.7% of slots with gate below 0.1 changes KL by 0.001 and cuts the + expert bytes read per token by 5%. +- Item 4's 10 points need R to beat D as well, so the score waits for pass 2b. Missing static + point: 120 experts at flat 2 bits (240 units), to compare with c48 directly. diff --git a/docs/research/2607.05147v1.pdf b/docs/research/2607.05147v1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2bea013b1a157f868ac0335c561efc328dfdaecc GIT binary patch literal 1055890 zcmce92V7J;(>EZBAc$ST!iuP%u)U(9fFLReQWV6(0t+lGEbK1T4yYjZc2yJ$S1j0j zSHy~&iAo$QR*7jGNc5&LhKQpjN*b5^;TbE5aSTyO5?o5wt}Mko1C*G0E{nLu7N| zP(y!$kU`Q|Acdf#zc?Bw>Mw?-M<6Pbp>bSUfHWx{ehrr(vdF)E5iP?dF$5Pxi{9}u zNx-2$5Ec+m8xH?KHHZQD%O8pZ1;dFCJ>sG2a7>POyp-lkFormwIgza=N@8ekNFGQk z$*654xL!0j_!DS%Mp@9f3QAxS9b=NEL{o0@Qe{LuaDp@XgM=_EWT7x56?bSd+y;71 zArY%k=ZF>l(gfN7Nem1p+DL1aB#9jx8fGu%bD3O;c%%({+uw5_%{oG*idEV(7@=}0 zU9Jdcu$XkNkjG;(hzjUzK2tzv@>pB}6mXY@NMe){vJjLBL5)chBq8$XXnBm1C@&+w zQ8I#)8C*J(0mZz0UHs{;@iJMkI3`8{wE0Fzl{EM-5acWIx1|k}D3oa2a_B5Bizj09 zc^o@7oe#5s%cQe~Kt6m1EG9^3Y$lsG(uNilD^^7)C1P3vF3P8K=`7TiLLM3)f*OgH zMoFLnnvl-o5m~-cl?-U_?-MIlMA_4vP)SUP#Lg!qLJ}G;lZ4WIVkIH*GJ+`> zP$BYAG?)^lstB5oBwA_*BVVqNgexRUB_al-D^bA5@)&5@S0Rp#g(?`XVj0MYl;6Pq zRT35}lB5ZQjKsl8MwMU_fbOb^hNd<((R|{ARm6Ps_i%>>h(FaBz;K6t2g2A~0S}c| zs-QQbje7S@iWP@MNmR68Nw_qokL8;KM=WX5&_0&Kc^*uUSZ7Iu)Gb9J@ktrt8tNJ7$v!DMHX87SmJ}_EQQ9ZEYXO9!I7AlNKfRDV@f0vF_>%yn{5YQ*eR1?peNbID7&bu=r2)*D5S9j zdZ8-C!SZ-jA4^Mz-k~A(Dx%}a7D7YR4aCMPWJL9$Aq)W{-K8dfMZGS&K}MvhwD)X34+J~gtdlu=z-pqpxBX{f3i zd1}p7BZp9AHBu?PRwF}0&DF@$(0VoUC?!}U1M9>Z=|BNiku|ckG-i$5Dg|02OH-fL z$i(WkM!L4Pt&y#%d~0MvgH^h?DwhBRVlE;az_a;xkf0tDUJEs7&`fN6ung=5i2`sO zSpt@fnn**|54f*}RM6oB>eE4^Gm^xy0H;hqeoaUM7>c#WR)()!F2nUxmPU0$!=|=| zHR~e~ULg&SP$}z=aQ`R7V_oz+hDx;#Qv)|i+yA=1kR_s$hDlL}Q6^d42E%U&RT&=~ zDG5>4U#gmVBScAe99Sh9tTQ0h*}W=NdxYb5e_vvMG6E?xsJ~bx>0=32A>WQEfdBe3 zIrc1(JqJulKASJ#_GB{cnatYcMUH;35NjX(kO8BgQ+@P9vOQ&Esx2XGW1?0FQGh`N zCKW?%Q#lxEm}Rh_>SLLRj4rgc!d4qzWuc`UW&D9n08>IpEX%6QL$tJ0e?beRiHLlL zt5iiBMPvBT7z5-s!99A`q z)`!{^aX2)_U}>lls9>Sxp*M|1u1i_uij+kzLs{hNlSM8tfk?6ma@AQ?KVp=spQ@E) zo<;J=Cdp-!gtEyN*<_1QtGhH>3i1l1Sv$Hr_lu8_#>GpJ*>9~92g@Y%5HVp~(%Cj3 zr>a&_)l{pFF)?zY1~y4Qo1~vj(hp6*ia~{pOlX8nasz7<#5bFykV8@kUk*^n<6~(Y zEK?6js8sAEPl9sj3!aEaXAAf=A(urLicq(Ci4{;UB}Im}L@AF~gutSlLvpDhG7Z{^ z{!UEaR3Dtg zFrX?TM;l)~MzD;?f4HQmxTJ)*B%54yXX2)*)`*2q^#MZ+wz?YQlKgW?R=FfcT(Ukc zNj8_Hq)Pn%+#0H|uGV-YH$0LX9?1=lFMn;ggj7Zfodk?bfP*gimtAC%F-j-286Es4P`6 zh7bWM2m#52fMh~IG6BnGG+|&SqJN-~Iw2%Ps*=#f(r5{rrwtzjG*xesl!k!hL_l&P zBsme1YSCfFKL^*1bBMA$WhDZUFC|Z=@x3U92fXoiES$0YZz$>^;s!@a3t4<9m z01+tw5h(x>DF6{ksfeVsCN-#0L`|?HSsoJ#q(G~LyM@lJ)-bBCTEileaS_S5h-6$u zGA<&CVlmYNtg7$TZ9rdJ$AWK^&Z*WIqPkjREGEeqi%D|EVv?+}m?Up3CP@v8S<7f6 z8(~6o|EQ}7wM4b+NJ9=`(D#1c11Xh?y}v|~+OHQ89}aGy@Gx-*C_y%o;o5qgxhcLj8Bw>K-6r5TP4m5_7ToDTXG*Wjl(L8Y`ez0gP^uvwD z00O*`p$Te{#iw&vLh#b@=uDV<9F~BN+*53pkj{j03E!)ITJF-AC^h9s^=ha#LiN_0 z)o}3B#Rn5>ml|w>U$6e`aQH&H5VRx34wogO3t%SIn;q2YwX=h|Pu*8_8~m7dAEMWX zLv5&MhIr7d!=$PdJ9ij6R2P5_X7j-33G*JT2cD2a7r@x7A^0loPna?kaoDE_rUe*@ zG8ObY!xh}262ccMK|O>G)}&=UAL;4nyw(|4(~2dmBC5vPs-aw$941UejXG}~8c{m0 zl;2|fiY;Y>HwXjUIO*s!zU`dVy|SC+y8V(fCo|_87@wFs=f2Am$;qN?Kffj9>OK$d z5yxF!@-Nc zHX2?0&MIjq70Z~S8W?D^4s!jRPEtFxBpjk!H?+|PwxJ<6s(jcLf<$Ct^ zc0Kzno;I)1=AzZ3mWH`_&JOb*^HcH8V@`MXOe_D2WsjZa&Uvw&pY`!IZ4f{rh5^7kfOYT}vGu)~bJ?HxQ1d)Z`pbM3+nKc9R3^%{5Q*{pcOQv-r`?F_u5u=YHFD`PafFEjyjE zoqJ}CDx1XIHo4fo}zoLMqWBc4UZnC8Rj3$#*Cm(QgM@=~B+e1|}F-pbhK0;6R zw!w$+PqTIZR@^=OW_kLGYwvEPD%Zc1>0XbzJnF5jY<#mL*LF11Ym)a?W+R^&^JZ7u z%F81ze6#Vf9emtp#??Ucit=V+QND4rzk82aBKO$QF|kC@vvXKL%A)IY9ooj}OS|e0 zc{;~+uXDCUXgW2&rR*;~^G7~qyIEsD_HqetHphFp-J?O*bJ$zn+lK30?tb+BQ=`v@ zcDp`oj5UuIwfuhhN=EoigWO+*<&VDyRfY}Gp*OF*`L)B~{f-0Qsd5#ITpk!6eW-K0 zOWEAu;t4sRwb_3bQwkabc}n_b&x|r zO5T{+x4y@fwCuQG*!K&P%O9E7i&Jch7kfRs$zJx?>%3df7m6O|Epu7>*0V@|_ep<` zCUfVO$ih>`b!*-4-r{}@q_dB&;l&)g6D2SDw%+y2;mQ2%FN(J=Z#lo`N#)L$JBN*1 znvuW#_Q<~;|9J4_y1qqD@aXeQes`i*&IPd)P){> z_w5>W*5KvvpWkM8b$j7x-t$&UqJvK4tWos$eVw)n^Lr_k8v~Z#>cw?xT4bD_+Hgr>Q~Ftt zn`L(MZN|PIv@Cth%#PLlG{rd47>3?oVtV~uu zV$Y+{k|{h_d$YnJFHgvFI=*etyYJG$oknaqnl}2mNluFfQJ zdzwAo6P*@p`8uwn;epo1zGXA+Z#(`=I;owE$(NNK+J7QDq_@!}gTs#gMcr)obw0G! zaZrPncCsJzzl0GyQCviMUW?a?v>4{=$%7*^%34=6>?7j$-!frk^VOpBk_bmeNkLqO z!A#weI&*J*f6}(JY~`B+#u*JS2DbTj{du1XL(@CO z`?E447~-A{mzyxxyNyZ^ra$7`QtXRX&E!g?4I1gF{7OYD!cS*%C+uB%vWNAkjRWrY z3T^e(J94mU>!MiwrmqZ^?!S6y&6OJ^qi0xm=-0$2ZQ);>xJ_2Amg)6yv_0vSIkUl9 zrQU+3KN@#*~d+n@*+n4~`gVrfiZEaJ}r{&sOJGFIzvO*Rs}Dt4=u&ayzlR>4{BN>Ep6m1wHHY zq?g-K_r6Xy^P=bYuPqqlbD+?{e~RSw_eMSomTtN!PI|j|CnGWDl^6wEZOZ@E@$wD2;r#|DZjPJU?DKHDF}kykHru^fH~)su!#&$Rh4i!W+1u9b{n2}O zn_NoKH7@;>m>aLBf9rUgK8IpEFoj_+8e9yN^cYuit-Xg!(WOk8w9A!`>3T;u^+>SM za~g1O*u_pGvLweEw8B9~i*d=}28ga^QRIy7! z!Q8atjRT#m77qUO@zu2z*%lTZJ}>+>kDlq!z}1ueVM*Hb!O^^HZg+q1Eq5HsNZ)2O zec5!Q8I4CYVt4Xz)Zf|B)n$sWdGm!&u2|C=8?G>29N2B0*BNefoK=Z|PTx!Y^K6x$ z*S3gyZfQB&rgXo_a+^l?3i`b=8owxc<#yAllZv)|N?meS630oJs<-89p5aT=<0E_| zGmUiD#2{PX-{^Hha@!rFM>#ha z$UGkz=q@W7f9Gj%kjt!BnSERPXPm#|B;Hyc*dy+AUeT%2hOcgIy8ZFqp@WU@-t;?= zzHy~nkF0MVS+jb5_SrGkFQoKZ_Tx9N*Oe=D6csPK=#+@eRFM2g$yL z4c;egwqEQp=IXT{83nD`Mwh?k2EGWq5YZ!MM&R?^D%YtSnSMG0Bo0s4zdEpSn8A4A zU8^9AA@?SAUexH-u>CfcX0&G;EXD07zA77Qb$|cq@b_LXzDXbC4sNB}@pD-((Y6B< z%Dt@oe74@{x@TzSm{Hze_vYOndM;SB?M%=!bFXL9OWj}Ui>yqi?w~ar} z=rQigldgZy8FHUqHy6bhGn+ov zZFl70&2N=6FB@f_`TTj0poQwwkwf<{Zay*6C$7jCcNG~al+_|t*Tj%Uq`E6`CVDaE%Lv$ zfEQugtBK*wGrSEmCObzxO&;3TXgt@#DC>OXRo1JZ`9Y@gfPUSIYvD@MDdP+tgZ`2P zIHTh|zzMC~Rf>2xs{*NTd?iL1iyYA*$<(r(bYoX76$y7VB_LP z;g}Q)Z!Mtpu9sK^D^iq3I#y_EhZH>P;2A-NC@SDp)W> z5M?azj1VsRT0HBNBuE!M3-5VoP>b3l+7AQY9FxfvF`0Y`lPTcB|Jd+1q`}cHrho_k za>#T(`Ua&-)W`TZnw|(?8XZ}R36&<%9l?W83)hHz4X*JX3OCeF-nnAo%_bJ!ZDQf=CKlddV&N?&7T#lG;Y}tM-dtkgEg}}){$b&r9~R!_ zVG%u!w&uu@$ika8EDm;o;++@-X=rS+@LmiHZ^p3jP6`WerLgc`3LFT;-;-mSg||~! zcrS#7H$zx>Hw0#-di#upw?^PR>z{6nvGBG^?K@)xPwHVyZR@e{9t?{|`e0dj2Ze>V zP~hYt$r9c~Vc}gA*ht3d*r$s(L0EVfgoU?3Sa=_Vg*P!+co&0(w=r0FGlGS8BUpGl zLQ6DwZ{pvG2JdUsAsS6>P&FFyCJ0M_dlm0qu<-T;3-4dB@CF78?_jX-UIPnnHXyGf zF^ceZ0}Jmru<(Wh3-&vSe7qgN!utU%ydl8CI|3}cCBVW?d=~cNv#=YVh5h&}?8t|i zNz}{6Zg|)X(nlLEHCdrH^D(@#sSSK;2Oq;L8^bFb!z&xZD;vWr8^bFb!z&xZD;vWr z8^bFb!z&xZD;vWr8^bFb!z&xZD;vWr8^bFc>Lo>sH>=qgUfCF4*%)5g7+%>JQnf@) zVdQVaEHen^jOc8P=vpjdbpQPUUQ_B=G1S2#8W}YQ zI942NjQ;3=3BeLZbT&qGHb!(dMszktbT&qGHb!(dMszktbT&qGHb!(dMszktbT&qG zHb!(dMszktbT&qGHb!(dMszktbT&qGHb!)?tqA%#7|}Tx(K&cym4i{7gQ1v%;gy5o zm4o3GwwTCzFuZawymBz4axgA(FxGJ})^RY_!44)aPmWv;w#+#g>omKbLs`9lKDs&B zQm&)x5e+rEo`WHkgK?3Av5teWj)Sp|gRzc79aq9w$H7>~!C0ro5_PKRU$BHBvksO} z_tazw!z%~FD+j|X2g54|!>bml7;68;#N%L;uANi@dTWx35uJk(or4jbgArYuL~>I8 zK8YkQX`TqfilGh?Yqjn;Sc!1365(Ja!of;}gOvyeD-jM>92|`P9E|83jOZL}Fl(`f zjpu*E8b;|lSgX~l;$TGQU_|F&MCV{c=U_zVU>lu-5gm5+iD|9H0!H_L!va<2O~NMBRU5oItL><2O~NMBRU5oItL><2O~NM zBRU5oItL><2O~NMBf6HTFuMO6QDMbUhp1|`;y74|aIg~LU?sxAN`!-z2nQ<>Silp# z%*9HCiIxfaKF2*`8#yT#>IxfaKF2*{z$x7CTv5t$e zj*GF5i?I&8spNMU>$n)-xEQ3k7YDPe~r0k~};md3Z|l@Ra1?DapfAl82`x4^K%Ro{~H~C3$#C^6-@8;VH?(Q<8_L zBwV2;>BCc!ho>YDPe~r0k~};md3Z|l@Ra1?DapfAl82`x4^K%Ro{~H~C3$#C^6-@8 z;VH?(Q<7JU#lx$&#Z$BE{@LQG)w<_l6yRYL;9(TtVHDtD6yRYL;9(TtVHDtD6yRYL z;9(Td8kHCk{>`YwU{d?2B)X%9qT*rT;bGw6Vc_9m;NfB5;bGw6Vc_9m;DO8UcSdkJxq{GQ6@$>fVHM+79jw;6s^DR)<72GjW31z2tm9*><72GjW31z2tb@P; z1WSC3b$pCMk7Xo-T-1i!sXsX2hLVyHtD z==++Yz)FOVl?dGVBzlRDl?Wdz5k6KTe5^$HSc$-nFZn%IB7Cek_!#}QSj6c5Z+aUm zhB{cR)wj#XN`#M<2p=mEK2{>Ic}=o_l?Wdz5k6KTS}b6N@^4tcx~L8oYPHt**kI;k zUB$-=N1L?zE^|o*qTwBbEqFo_b&yu8H6p-zOOq4<)?vRdrCLbAe*Id(n3yZnSXOV0 z3+nBIsHItKqSfB4rY5ht@>K1{tmaSRq7@D_RpV~00IPDXE4OeLMSxYiRyy3p6<}4b zl@2#@1z7cKrNgZ&0apK7=|GV{?U|@LQ9{4~CDGbhLl?RIS%Sq! z>14lZrXv=xiJ+NGa6op4W-`G5iG7;MM4M!PYTi&LVC^6I0-~(l!sK!l+@T__Ji|yP zBZFyo7(qPg_S9?_0y_z?-J+Q+gk)^SXeNtLG8#qR5J^cMVh5+a6e^TP+O67z2(Z1Q z#gC>guIi^M20)nB167h}@~Uj9+Eb;p)^vFu(4UBk&0xE4Ii>CG&r#Z zq;6b*O(8AnHPvwy^#r{@0C{h7hzs26^74Vh2%<3paJ6uP#l*GqDqg7X;Rq5bO0g_l zu7K-n(dhV@0NY$zr?Uju03(}#C|z)UjpeiI-@w^f<_CDF-Bh@O!Q7ae__5ubRN6O9pZ)6^0mbquC2SRsy< z!)eF}sD-bLl}O-{DIDn{`-CV=u^>>}ts0PnNU1qB$#PmZdC{>-afDbAO7?^11l9Ul zfGuH69%7CUXEF(@H2alco5(@XgY8(VP`E^OMMUpQOgh^5y_zn zF?o|$B^AqL$)OT-8W3*%LVB1Ag58BGQE7}-B&FJ%2(YcJHCi++fNG;ftWZehGBLEy z3zo@4qNtGq0oaIp$KsF>2<3%hs-b9hHRV*JxBwgJn74Qs2B;!dtU?|Ror}oV>T&f4 zaE~!QL=~?fx*S{fWF;YDDWWe7!eT_L;CyQpMKW2m6hi!x?wi*ttvV<}680{}>5&R!CA;7CMvZ*ksLXs#~ zL@9wxPDp&H7{+sK1e%a|@rEfv{lJ#Vl}Z^zB}BQJ120?>9W6%4j~95jOoTWkK^gmx zg(i3q6mpf~=ujLPQe%KnNl*r_gKKgnz{^`LiD^2Zs?B#&B9RheCI&-v$0#6IiFk_% z50OaGKNJYm87iWh;{B~E!Isd_fDn5$hQJ8KEwI_MC*o048&O~IL*;;vfPWzn{e~b? z9Iq71q9KK(Re;y#lxWa^;KxeD;W7zmInY2bX^`dt#)w43D~jHO8q|azsZt@9$WUQT zbVgrks;W9OCi)I_L; zOsjy1Y&aUC5CcDCmM`*LaKO#sigpOMGnEEkpZg) z2np&^?EI=zG6L+{((03cSp39@C7?!OO!4HwDu|e(fylAZGKF0NNZw8c_pKmWF^nw^ zg!ZXn)7Dyt0Q<{GNRJ+bJaN!>oMYtXg&kotx++b?{gW(B+Vk%Yfp~}}J`7c%W20Ju`3U&JZ^bhfus;T&z zH3%uU8$q8^92yZHg@z+6a-zlIQl!)=>jK4kfeV1D2nb*o9uXU_QYplt5_wn{;!@KV zQljOoR6R-AOgk{_;3LXohntECh1DEt2+d6dTWoBu zBsvy#7v+#6^XrO7CB*(YErI{TqN+xWDWZCnG!fW?=J_FFSVASjkf>!~Rxc~0oPd}_ zVp#JD4v}%Lkf;<=NrEIAU=i`7Ik=Q?+ff-G0}w1mD|c?JtO~kdNNBG-g?Rsw9Jxw# zDH7>5VnvNeRU1>-PDNS^HK++2(3r+x21TM0Lsae<3HXc9rwn`i{-BQz4lFMKxgacj^;T4ospyPzf{?3GFYWoS2$f zpo(Ha4vKIHbO!w|iB(9#q)Fs76jB~evH;n&XjDU_a_BZ}+z>xNA4_Cm8rld1ianv2 zL$t0HN28S;@;X(mvW1jelWe3mK?t#rQ%mmuu#o_0K4$kf;2@E zN{mk>crS=`u2NgPT5e#f62fx?D_e32N#GL1MHPcUKkfokgr@m2hDk5M72ur{ywEiGUY6fB|l( z6r5-hMFIkQ>^`RIMMIMpBaV?H)0S8@!0HzP4YyR24&! z1=YW}+9Fh^v*0Tsqe1&IqO_iZ%iA158NyY}JXd{UxM)wdzFJ{1Rg4Y?b6- z_d!T`Piv9^Z>A6yvlc_OFCD5C1`${8AUz@FL9JGb z5Dd2Ns%0VZP;SX;Sx89O6{*E^?F)cvg+Z8Pa7|VkLLBPmi&HZlcC zk#eY!DMTqe!^spRMarv1rl1jsy;z#0lA%^>+JRWRx!@~zu zD{Lb4yhZ}?I+YeL)jk$c-Uy9CY;*$?8vM84*OO19dQc*4v0=4w5L9z(1e6^*ZskCFm!(tWT!9-Xk9CAq`Xq0w24RxtG zbTk!{6Pz(1qCheHoLv3A2KV&y9`1$A-B79A-OfoagE!M$V(?pQBD~Bd->7hg)ea5C zR)R$#^+p!FgeAhOX(GrH@eK-}Qx#eooqdB)V7_v{7%AG1hXUk~z~=%xytLLNP*Z7D5$LNC_p53| zFosKG9AO=;9wIgAAnr60ovPW0hN{s_)?6()RlUjqfsN=e5;*YU5k#GVH8_BEtuf$S z<%($Ha3zc|?RP%NcMhd82|9g8v!*lP2~D{=$~qmMj)5Rs#76^T!sPv>5Heka7yD$g z7i`20iH{D3u-j01)xmxg$E+%}8u}$>5;4`BArcvl5h8~FCt{mvhdxWB`i>qxcveS&@$FAGD&4b|>)$xEt>_W;uyY)L{?YT@ zP9^$nD-CHdd2;e7JN^mJZB~KO&z}k^x^Lw7>ay|R-OXFBuYLRBdG6HW?az;niQ1G? zeqzY3jJL05za9If{Zxy-d)_|2_qOs#z=Us4zy56+X!3b!Y+$bmr6&su-rRaz`q#WM zBkw$C_Z%Jm?(Hz0@%BM4X%S5>90+)3(s!-?v3GARM|Ru0Jm_Pb?#&kFO)fat8$Y}f z(!S!ob5MgeyY>VGH*lZS?tQx*o2T~4XC#^LDjc|_=b(!Z77R`Nt8jF&!FYF}@%WB6 zEXF47IRI_YY4zzsZ$doz;4&#bmzDt*N}!0qxu_+%I=I zC{8<;;IKthdHW=-eEF1^=(QaqKj?1FozQ-WUew+01-p|M8kf1etO%YoVgctv>ZDcI z9?zZX9+lVjO5B_60S~h>SC>4#vhUmSMALq2qGPA3mVQ0i)vHP4)(a#rUH0z(tI0)^ z-QPvQmWzYVnEE|)-ioyYuQ4UNMz?*~9aQ5c|}%S&Dy%9;iLD{Miq!#8J#H}vPgBU1*f2-@rjwcHsuWw zWzBM0;_7y{UGLG6(JR@;O?5={kJ-x?7Tz`JoUm(C%xYo#I5YFfrgK>Sjn5XG>0vzM zg3Ro6hFsUpc+1_z(_CI3bnM;gLNIH?^2eF+<&$^JmdfY<)SJ3;|NMuxeV=a~y4vjh zwVf_}^Uy-CQMV`iMh?vH8g_AB@~Uo*ZC0igM0#4EzH)A5&tS6_O}E`0wfQvrlHP|Y zjNOyCHy$MUv~^2A-y>(_OaJ%TYX(o*J9=ECxqf7uqnUP3R&?E9+vRzJX_q~^oj!kj z!ZzbRe2{+1eZT?J>fRY%((6{8bbWPOb+VK-y^&>kuE4eJUXRxUv(8_ZymGeRTE;k5vg1eg zBIl8f*V*qcv)Syp@b$comkqNPEX!>&Da_NvW|D{bh#cmgrd@a@A%814zs~>MImg&; zL-C0*Vftr+KXeZ1ZB@Br_fY+Ftu{Qad}JQcLv~hosLSbDS*`tcj_bn>e0lGpRnW`+ zt2*a-U%zwNS9wTCyZxvlQb)M$1><;&%OiIbL^^%CTa@(N44vT+wSf10m z`C)yRA=|x;wlJpMdup)6^<0B%T}w@Ky-g=~%W&US)LuWN-2?jCC*mJ7Pj0&WM(EVp z`__!L!-l&pxRGwwa8u0poUcEgc~0??E}~nNKc02wl5I!JqnF;U&u={9d-<{#Gxm?| z_2Jz7X>YAu%a%RnvUaRV9=65z(wfT;UU%?%VQ-N%ob$1V2W#1N58j2rH|869b-U!c z^zHp;;pW^cVa!@RnjweD{^zi%cGmAAFS?y*aKZtHF7ToZDxntSXtk zz;1L>@ru!(4j%PqmrPqcquK8C=R@APjhiX9oT{wYzWSii|gXvn2jCBYu*>DJlC! zBeW%0jkN@MWDzDR^|>SH3~VgnD393oz(FgiOfs^g2)l(f6|ANyiE6{fPPOvHds1Z6 z=-DyiK`9Y-6>HX}VPG;)ZKSnB#oYw6iG0jVgdMkJ+)WX7$dZ97McDaDy|YCTM7(H4 z8HOYQ*ypNAfZCMBfj3bUNEC?@MVt&qkscxPBTCbVUdiXGqd&3;JDd`(Mqj|1v$nn< zjC1O9;Fr4BM)U<1Pi=gNu6v0>p0-UGN(K}VMjSOPv>0A5WvvQJ0X zv%oS_8#3Tl4=9$wGKG8x|Q9$q;0a&Ihkv{Ylnbbd!(#P*Sez6viLR8A1@! zQA8y*(Lx?w1kS7a6Ro+1{$CR<6w%>*a5d2oz>6;W1Cq5-s`_Xq_HX{rL(zXOc|Mzt z0tvtcV>)^=3PO0%QK%xg!Al2ZsmBP{hR6TUY(Qi&xP~ICVuJ@SW`di&{%oj07DW;y zmNwL=ukD!RAIlvgTG7Fuq!zn~Lx;_UdXr16c5BC#|CU^l0B-hh)$~Gm8ag^dTYrKz zJp$VDt$DcWkEIJS1L5^A5e?!x&{4!02slSaCmJ9a0v$!}`EA@L7R9y8_dg^T;@tuK zv8u@B@#!dFP5sH$@HA8#z16Q@{IP5~JP}<4CzmPNLKG!B@@LkcSWW+j_E7lGh=s6B zpbg>mF+30;%n=8YX)-zOgK<>$dVI+ zsRV~BA%G^#FXG+T-&=tbhs0_hvHvZ>5Y?10jWNMoP!L?$Qm#M2b*r^HkHY@3lp&fX z9i8N&hz0``TrYn>blqC*e?T-?qC6P76w&a)EPCvt{*ta+so{yId7J8wWz1%B=;%l- z1VN^w(=K2xz;77Rd=|)_`}^Qa94D-ODF3JQ!WtR{M_1De$N?(3{^ZuJ(P}+l|K|dR zRUeA^MiGpZ270xo{uFEO3Tv-S{#%NX=70<1YJ$6siAUV@~&7nuv6Q^>n zoQPU+RG1v#b8tTAU9q5WCjXnq&M)_!j~}_`Fz)m7v7NI#I!uxHp3I(h%A+!?`H9A5me=ba6A zRUcm;G5xi7=ZpU71A9bVPuSQrvD7L{Px9<~v`2neuf@4}8#Y9(h?lpH8OA$&fS2I) z>S&>FpwE3FKka7Ikb8>4(R*U`zGtNUwRz%K-t>8|PfSfLwY5w0>^^g?vh>LIdwUzd zW3T#gZU6Jgy&Z)cY}3YcF}uIcl)LiYyVDbYROo*F`C(;cjHG$_sbOi?02>D=Ygtow3=Z`z-0<*UcL?oT%5##A5b@eho{D zmYI+8@hs(anf{tVOSAfT$b3g%V}|*!2bN>TuQ-0j`_3x;DZ8vJ&8BlD1K&R{>1by% ztK-Yn*G!6)Vf-U|>FX-DKJ^LTRVs5TbxRv{S@dF(xo~d&hJJs!n-qCmb^fGpG{G}^ zNby!?hrM6oWJ@>kj=acPyYi~R<3;!8+qSq-@iei?t(bluQ||R-ENZl_h}qih!}RT8 zk`7^O{_goU{A`!)-G&`v2EtV>3f5Mhm{Vc2+&Kxjy&4i#^5vOA5iL|{L!46*Lt+QMetb zr$+?tc=}M-{zRHj2&+Kn%3QyF4-Pkw?n=1Y*67fN4{avrH%`h~-pE+NeD(Cf*wcr` z=9+q2+MZjzKX`Inr<0xQla7f4ZWV^Lk14 zuN(JYN)6XGs~Fqmd0~r5U!U1^GHCN9A|i3j?(*03w(MIGw3@#z%&Oza!Mbrt4BaIQ zvTX+)b7wZvd(hrfaa6a0QF>*^GuOf1HjOVXyO3(CH@?-V9R0ot4DutmSn-b=$9-#Zd@>Vo&Fll=zv zxa*ky!)~@Pa_h-kitz)EUz^T*AB&Q>NO~y+2He}xgT0=F|fW; zILfSH?{Bk~Ub)QES&-aj(A;l3%ytjWpV+=+-}fe%6Iv zr-gC3DaZ8o+|su__DDYBV_3f1a=%ak{aeY5i6=Kaw|o;a*Y(*FH%{|KqOA8TPv`5n zh$0h)@cgXLPMGYF|2pZCM`2Rdqjpxe?DwR*vyLsD>UxDa^r(ul)cw}my@q`+UwYR& zL-u_`-l$FXEDLMKk_!^+xl*&d)j`&}qg9@9$A|XWH@^Fs7CIIA1t1GYI$>nP6wY_afa*1N41_eyrw=sMAcbH*!*J8*1(@JOG1>$3^ zK3h#N?e+HWOxMfz=bG_-<946zez@`FraKg0&L-b4U7zwWCef1rv_+==vN@f{Hxhiy zZ}#Q5(?Gw=oMBzN|7>;2aCs-2Zc}@QkKCK)eI%#oui$>TNj^l|1|b;>Ew`Q zcjkQE*^w)ao^D?+RTO2O9poN1xqoY8i%lD?D}&_A zbiJZWO=M&HzuRiw->Kb>_@7H}{RoY9`!-~caD&^sR!*CPXZ5%}eocpC+iyENtujdP zejJ?JI?6xSuXyJ9A!UEPHQhQ;A<0ncC>B)?N-=0+a>(Aybmmzd9g~LrD-vE6?GUi3;h)cHx<%{`C1 zTx&eLiosdZtDeT~cXK^y{G;$NOtpSd4Wh{Z-7MlH^4n*PP9}+WL;?rHtHg z;gR(`(b4ZC$M)gpP)M%!3rOfOr}Vdw?% ze)CofZDr*-exZ51A=7yuR+?rQygZuEdopk=b5pXbS3s!q-@BNv8Xqs*c~`%P<1uJZ zhGB?9s|}{>Tjc2$*#A|Qt-s*wq}7ebne%iHU4PYP+mXCBTO)=%89%z<@XZg;Z}iE2 z-tXZ!ql#gfS=qU|{*U_7Cw^*}->t2)#g?$RL8%FC*KAqo>D`Z+9XBMrQA1Hrr^8b_ z(0XhsKa%1yhEo)K*kr=me&$897c{&2{_#+~>GZ;LF2@_}d2IFh)9r^VukOh1x~9{? z*|PI71#j*R4ZF(yz15QyC{m8@D-XF-3B+cd5jzZCcWc z*4>+|9p!n=khemhb8>%1Jad15LGIa`tG4v*`{Ze_&+o!F+A^Y!4F9O;xYH`L@s0(3 zt`&}2viEr7b{Sxa6ldF>WS0f?j(wVu{_(uXa5VQr;FOL(i~HpqG0P~EuQayH$Qku(`iNSW;vgm?s00I zYTjJ=@n(U;75hb|M{Lh-jvH?MJN~B40fZ_4}6LpBd3|2?;@KqI-_e|)lau|#o=wv(Jr5sofA-u(Vq3F217Eb+ z=h^Pu;YV*?j>*~S5TLMfPyYMBtGKcOFQ(p^F`>`>s4tvPOGY!SCZ&AXnd__b{@u-) zs#!;Giq^O6G;wV7!%X8H83oJi%$e)f_zryDJm|~kLCJa-SqjF*u^mp2US-_rQs1Nm z-;fdS{d;?yagEZZa)M*fgQA&6Ada17p(*UtVRdTHSH& z`!o4{8#Q_sXPc@lt#s(zB)9jSUEBDnyDHM@C${an>Uh1&;mo9=kDCnbdZruQ zdy-kQ4gVu!-MH+d1BNVn{$a%8&24hceFg5BlVj)nW%2oHY4D`;fA=;SGQ%)yZN~-g zzebzIr@UNI{BGI&o{vYAJm}r@vM;^$rqcDxr9s19v6VRow>@oDF!}O{yDc}3Yth5- z-i>AJTD=>UTXax-cSy&l{cX#4usyC{*f`1PUG!+q2mfQNytR+cOgrB^@XW6b^V=*- z>D6TO;$z>w58HO5&jtN#udx;vTUL1dTKq6C`cvN3tae96ri>kA)~|I=u4t^s$7ybt znU!14I9xrqv!MIJ>%aOmAC;70FsS!6p92GWSv|=5CEGjb+}d&5vYOakNOc=!vVZjU z(cibM&iq<%b<4Uj#&pix^%qyX8Fc@i)%zFy7Qf-xnlDf+Q#_eJVQ+8R@;h(46(pEC zRD9^YRXXnHns*rs=Cq2{xpC=!<9A<9&0S>5>^q}uP4t$ntAx{?pS@e%WqRX8ge!tV{e#qS}w-)zVzi37Ls$Pq~IOdkpPPKSC z?p@LTJ@=w)oA&qmx?r7W^G8iw<1cKybmYPH>Dwcvul15|pR%1gu5ygc;kIA<-^m^P z&UBHj$*+5Qy(GN5b7!+Fe--%;&OW*JH6z7vs9wg&O4mUvJ9H3b_+5X~$>F=jdxH&s z*K+oXxFJZn3I5YJlF?ITB7EzzKF=X*oB<;4C{n~}FR`m#i2ieT0Qiv6GZ_#tk&a>n zpj9?{=@bIb(cz}pZ|~F)7aae$q{91GbSCT;s7V#T{xw_~t2e22`wX?hDE$kW!nPMW z^8y>{;6&zdVDA?^--vGXQ04E-v~EA)e?us|(*|4WY&D^L*zSN!EcGX}Zr|X4wn+_F z8tCZwqWbg`7q)275w7~4G9#|X);?1IGh*SLP&!0cts<5$pre;e>Q8LlZb2%xRlR~T z{_&WGCzOB#ILpH0&{2#|cv1)c70@8$JnW3rV^9;fgKHQ6|7VUw;21~8i7-cKYaN|v ztv^SaenoAR#-3)1v464G3$Iku(Zea|ED<04_wdp!9fj9|2OD6{*5i;RarL@(`Tuv+ zLU1QKG^8dL_8^$x6|X!)wg@>hH>q@6mq-%J`j8-ih*LrFrpzMm<_s^!*G%-Lg7+RAY9}`{{g^u zCVs%#R`f&b^eepI2OBDIB7?=EqxY=fZ7a}9@SDx0qj$l68|{hv?%Eym8@z@?6zce3 zU^BWVpAf31{`|m8^>7M|4}2i)%B8^@;&c={6yDCFbHJSWeSYdzvlwSN>S%97+>k%U zCU}Jv_K0AM3!PtrEoFFuk?0b%$%vkC`+bt@R=EE)$pV;K8)4Ke2Ud-v9h$ zH@u?bz&U$@KytnNQWUO3x^lVV)HgRR$hwjW+ z__5`{FruU@f@)2~OH@SWbQORC3HXhgK zz|`f26ThmAUfRwHZJy#Xpna+O?5p>b zA>;PlOl;(`y1^Fi)x6T+z>01reJ*xmTD~}N>rxOufpdBHiL_OY_rkI-xtS*h#8?=a zOS(?YKkOd7wJo#5ofR$3?2>-D9cgi6;)kWDn{?@$x0|D9-fXY#4N({N+|O}628Mn} zTR7c4phu62j{)-+uCZDF?anuL)SfGcZKuh1=dF$#*5dhzd6Sk{XPP-44eZ(YbL+Vb zyA^{{j?P%UyQRL&c=@IHd8ws`&h+w%`I6K#{q6VCHLZ?~jp`I7SeyUptSGbLEz^aT znG0PSrnZnif8~C?RdUoT`-3tsD&n;a{$6?io2GMuX#v&Uulm8);PcJ}U-&)$0G z?KZm+&uLB;?!$KPoO8AD;egiB!+nznc((T`$UJ&A!vAo?DUX*+V}0+A=#*}e?$c?t z@8_W#0$xO|6d8%{^J4b%cP3tqOPs$++~;gStK_gfj7dY8(+5nDC&XV(a7&-kr2N)3 z16kuYce-y~!MJksV=S#S;f^WiT}q;)x4om9&{JGt~%4V~P0hvMed zF&kG+E_iy=;cV8d3)3DC@-%Lj(?}=%<0@Le#&Cb662*^pEc|ic-^2; zuX6i6jhiLiyKcw*xvh1W?oKxiGCvtImk1bkbc37|FVlndTNe)9sMD!e z;m(5gs>-gfBU$=WUq0JNU%qs{p~Hj5lP2tle75`p?PA-8TP7G5T9<5DXW>`5!~Vg< zlw&TmAJ(B2JDZjyKP~uXNGhHkYLIK#appbAsKaZ58rlnHSho)c(N8a`RLyW7K=jeUdI=?y!6Fa9j; zIAwx4$1_yt)A*>XTRh$MJo-5`3%Rr6fnDXrmyHLnGVSd&c$1C4(}2<2-|gYI>?qd# z_$+_>`6Cr@Uv{O*-|db0(+zy@o6X+j_dK83`E9|F>}y|2XB0jxEE#ffxblk6mGObw zE#BmBbT*WH>T1S4eyzjWz3pa={IuI&sn7mcZei1aJ*=$aV7DcHZpBxYPyAfbWy|rO z1CoBOA8}*i3+4xxqr!y5>|>Tpo@=1)xX5+OuQyBZZ}ChN*k|Rz#8&=}r^;48%y>3s zh|clIo57zOhdw!Jx2yC+Yoh@D^;1st_cosNV&0zB(?)%q5;|${hvH=chOeTE8!zIG z+SbT!bjG^AU6W(*zw) z9ewxO$klYJK}(%x?-<|q_glFnh1qLC({3{kY&r0%!&=9PT$Ann-j9@`3TUIK(7vKHht54tdi^m1b zKmKfTuF5&ueS5s7~NLodSTKr?v(f={x?gS==yJ(vVGz56W*?6{+*AH(N*4; zJ~?ounbncvw~Xw(*ZEi1xD0r_y5ZjgCapbY*W)}rEAsyl_ts%qZCm>=9n#$;4H6Fx z(%qo6ba!``Gy)u6vF(*Bo<= zagX~x=r1O4c^(K*nkW~A*uAZKly#83qqVBz?Bh$eVgJ-LKTb>f4&Tg1{Ms zTJf&6OpVn)HKprN4bNP1 z@@+yGmdcR1w#297xD+dKH3B4goh@;RRK+-ct1){_K_tD2$Om(lA&&~dauufcHM^fs z;SbczSTzQ=eVwq4U?(4Td=4HjtqTD?tPe)G{S@5&88}hvg^WDruzO#!&7m8$P*Z2< zBS+&-BkU{{9)9OTC{rE5P79rMiVeu)NO4B(&5os{(@d^*wom>Iv`Alh`!^FE>^?qC zUNesoDhrFztv7c5Fzo^093O+${I6h~HyUH}0unpXVfveFq$ z_x_`C+L&uSJi^w|8gvV7O46r3Tu@A8q!T{(Cp5xj2bHdbvoVgFN;xQNnAR`?tWpvT zuG>5ameLVT)~#HXaYoS0P*r6ib$WCBOIg5ISfE7$jSnLN5HRqXyUBI%cN|(k6)kA} z1Rn|NPbxOOU>6VZLYpf~Il4_knM03h)rL|g6t`YkX4LkLr6G(N;X;VNx$=kXkya11 z3BMv<*w%`r0G%LnN1f-EI>>hp(AY{EXo7*Gu69H*@ORHR#UJ%=s#5!8b~$x84S^BH z>t}nR$x~~@a^b!3DD6sX}QaC*5eeB0z zSLiCZ5ItFVDpat|ibUsqMj^TQuG!wgRhEl%}(`;?B7#{Bsn-`~zcmzCw1d9Tr}| zmHU%Q<<{y2itOEZ(UXeumSh@?#uoE#LbN2rl$12uGv+nHh^&;Ela+$gC699|vpat- z@J+scW7l1lwr-92A-;f@jIX(`LR!zw&EUrGCHKSgsv*wEr%V`Ud?FeR5Ix?OKD#0K zT|vu7RSbivlhvqX%Sv2fJKsqUPA3?GVKiQQ)AJN3~naIiRLISjb$X2ST`)M4Y0(2`}tLn{)6~IZSatL zdb+HgzRU|)TTc@Ou*ap;cDeIcpaLd(i{lG+qRQe!6z&!JxY{DHI6=$lQXr-e&%8D= zJh=!UND^>Bhk-F5hx5|ZS%EgNVMb!l>03Y&kaHFW!&Q7jhdu@)Mlc<_$p(YtT?wm3 zwuw?3N+uVbrl8JCh6wrP^BQ|!TtGG*&+Csq#VD8`$>={>#u!K8FtNt(F+&jtPArss zq)87AuI!FMQx79TfZZ?L-R4sg1wC|H7!d!KS6T@nX;9owos8LhJK@i*q#Q7Y8={U-2KP z9-=>@MmMl87jccC@$-URcvOGxbd2F0<0gzvMW*+fG#yhIZT5n9K=N=Y=BPbP3iBCc zKPnH>)~Qycz$q6<3;dHKdd0S{fY+HR17<8l!|Q{ylacy}56*jZUuOnM%ucwCX_c?@ zK6&xUMN=-41TrvSbwV0@gV8qdoD=d2L8VD@=N#cpq3H(ZJtxJaQ5W5Q!52UlkG{y$ zku3T}Q6xagvNE$-ya}d5UN5<~eG}`AGM&F0T0w}Dcuoh%PISf=NzRp`ou{!^?%unv z+d;b@99)K^`VqRL({m|T>u2!kJT*Z4POXp%8FuC`*$29+XRkj5Koh=#!6{b{a1Jqe zW5Vt0c^t(bk;S*jc2T?Kx7lJZ)*W$Chx{q$WGyc6A%t=<0q*rZ*g5zR>>;KtL;qv- z@C;hPo+yFrK-25XeTNMqVYb(KP}gY0ZYl2qT&3ev_U#8D4QY|V4R}o*ovGz?q2*1S z7OZ%cMKw%j4T|SOU92oHh zY6rKaLO@nRoD~?{lpaJ2%e0 zdE5NQ!2%U&pwRIhD^OPgX7Yem)Lpxg-*iKNhV^#|_}{&8{*7`YFggoJd;P$B+mQaP z>3gT<={Hf+{{~#3D}Aeu{sS%$Y!;v%cIWW^$^Y!n`}SW$)IYp1{&{eL@ejao>tASr zSpS30^fxirp9S|;Wc|BA(m%%u%)PST?$X~{=^0h$=MR_7eIxe*wP1EeK@ZrrV02e6DgXrz1-U;gXp z{zA03J;U4f`46ywoFoUpgt;@=|FdKGyZz@s4lFA$j0|+Nen15-8o*ok51{_MRrt$z zFSnNqK(zjMsI0)eKcE10*QoxxBIG}o!47PkZ}K^Jdmq0^j{o)h_}hj04jy;^_>r(P17|dFTmF45 z-#estekA`FSLeU6nBQ&!>;Ur;AnpLt0&KUeBj6xn2Ch|qI|+T$dH?_Wu>ce*7Qj>D z`%-T6DlTz!?C1gGkvq0PX-#0${o2l>pg-9Z-e< z?mBmP75)oJm<8aKFax!U-@)E;jR1*-yMz79G%RUiYwH9!avI+{1^kU|e4APRrxN2W ziwFoDK-&VSXn>FnaA>mM>JthbMz z3%Gs($bm_EMnJ{pZ$Em#M(?kX{%#EP?_dOQ0R$I7f#5sF+u(u#Zp+;<{^d95tw{i2 zbOx{&e{&D`#}EBBmdrplxFtvfQcJg41rVMAA_EZ2{(e3IbQk_Al7B+^XFb}V(Io$R zYqD@Ke-p0+0cg2zr}=NCfPj}D^o>7s7o2~We*@e_IsVMQd8qBU*8Tk8ir!73O*P@r zOAjov?r6DcP1(Vo?vqWI%KJe`s3^%4Nk1w<^L1U{g>gPoFz`5|C6$E;I7AbV2Gh$% zO@kWku*vkr(B1=IAFt!dv%bW5%nB^kQvSu#W!jKv14^}5p~r06#$jQO*KX?G;Vs;* zgT2-YrPf8BXD%g?Q5<=zwT^ilK350PzT5ufAH3i$#-H3Xhilg~C-*y2-54dL@RG?a zDOxDl3SQc(u-w;L7<1h9;csjx3cvaCsU(iaLn5L-cTrh;a>BCHg3|Ccm618tt|*#@ zN^_U7`G;@^35A~A&%4_U*TnFfSB2bl<*!LBBRcF4I8Gee7@qMXke|!it&Jyu&Oa&G z>J1;HJoK^Kx49Wimw<&$Ut}mfvWr*jb?Uvcl#xA|4&UD0s+;L~YcQVtvMj7`x7Z{h_)ynBj<9L{B#_DUIBZ>Z6^H)U@*gd)p0rClD2Pvg}62>$)Tgt~Kwc{SrM z0_*wFNdmQ((I+j?UpB(5VXt3+bAoI_Ox8|S+~^|=!4+eogEkFu8@X90Y&LOu??IFT z6@G3=eS>%kCm7@L%ee$JsVB!HWAq=l!Z9k!B8dxaBosKWa+|VxYA?IIpFQ;Y+)M>c z=9CnJYq_GKIc6Gn^q}_Dp)h0B16ApgDVACmUGI4?Vv2e;JUv9^0Eprs9i$DPnrdpA z$aF@6sEKe@iW9-UQo2_ObKT=wJh}CxH7^8I9y(;Z7)>&a|OxLtBU=YBiUNo|1X=sYh%FsS|ibZ4oCC53>lVjiw!*=GRPA%jD_rc;K6>O=;mtG0flMZKYOA$U-3$#lQ>6iPK6E$ zl}xu-t#b3^-k60ON^&n)S|CAwQ{=kd%6 zb^FcufjG05lP|&OHyO#?qTCO-Jqk~e;C3M2)@~-((hGB~g?O;{OS+rbz=J;MtsRumhq73jV{z3O zZ$L9FYg=|;X5mik1JTh_7d;eJbUsU3W6ZOubpOp)XVh{@JYMm*+jB6E)SX;F8E>URXeM#HpoeO zh1H7(%Xo85->o^vZ&T2gu(MTUc_;7Ts`K@sw_r#F}lKYNn@VYzmK>tDwQjbSN-R z33RPir5mcIU=cvD-r%A=e*IjaF3u!j;iZNr7|1$dboA4Jzyqk2{8pdq6@Y9p?oamK z$_e(Km12ur-?aYJ@<#zcH1Ug9qZB|+dM>rp1koUTaS4!G+emyx^0WJ z>a_ua4b4-@RS1b8vQu*;3wP3XKfz$NNJcwh#E1l2nzpP|CoAyDeu^21!89H%c^kD| zaOs1WW7PGD&sVH8g2^WQ-+01nWhM+u<>}4nddr`837+&m4*+)$GK}T$zCYdJ*DlGM z=QFjO(f#1?Bty@Fr9Y3&bRhTGg9F^#irVTPc%~d}eTFb0o4p;z^RigN>IkEfqjJY$ zwNi62TE9@IrX$WUF>Gq`f@KY!ag30W#I2tCXGZ%tkaa9je6CXIQ}tb}%EyFYtxJ&f zg^~iBPSPW!??|#RNI7TSG?4}-_SMB~&CVr zB<6oXFB~%-^V;9u^*}!QWTvNTky@@Ni61O*W!nQDBkgz~sOTT_kz7Q956?+x|TH zrVST8W#*NEughLQYAnYTt*MM$Ou6WQy~I6y#b;dH$?T7*hB}L%;j>;-r}Jy6emP#v zRwURYckbT0Z-Q%F+!ingA7D5_V*&3$f2uaOUZ54qB~wW@7*RMI>fTOaK>4){j^H4O z9h~*P?63|Rmk(WX;*L=pe~v+0G}L%bb(oKUXL4)X`>&7*V;zoP`G^`q$F*9PN7`Bf zdOh$sU%hlx`ZD*1_H*x7_reL=%pF?#944cjll5Y#+O~~8xBe2(em=%vYCYz*<1mb_ z*T@-;e_AeQ@Zmc$%&5sZ#|r1b$(!p`dJfhoEQhoqM`n7d z^iXV0Bz*Z$VUhc*I5-aQ-A1G74=-o1TqT31^tzwm7w(WJ@Y=n@2sVW5v?sJCMaE%g zD(NRmsNfY%4@e)r$48qK+=Nfq0!FQ%Y3EF#L{>IU5p2$lE53qD)NUP^BkH7yN^i$e zYI2X&j->=&#H}_Vj99~lst8%kvnMa)^F$6KVk#bq^A)=*|47ZndP)W6>l~w&@{978 zv9Wh8?;qboNGk}1Ls1q9fEDg2BYc-XSU_bU#lMnQ7KR!JyZW)*e6&`*UDo;JwXiNN zMOrL8h!)D4iYU%7(UjrJSRlP9qEoNF==3P6K2XXWM#++#ZJ9h1^t5)p%yYUnuz|n} zWfo>!ZIUTU#?5;OBe*}n@t*ospGZv#1|Gp?6zOwT#teBsOEC`F5Z)456z{f2r}+J$ zWS)C69(?E#T^IJ=a-y3a7=oYQ$JTPav3E$MPasefSX181Hh?2*ep#i7`&5uRL9g&# zt}JVlCAsLyU;*`%_RGGouhA>TeWRL{B7^skIk9<#`1k40S|RIBt`8Zusl*- z21SU%b*97Vc`jRK(|XHC9&^hH7&245*pg6lXjbt=G6&tbV;zzB@}Es=eTHJ}p%LSx zYunzE1nnnHWPZhFj6f<$Eyw3lu}?z^M8c|(P)pM%Vhw?iXCUC9aFSOOF-z`gHzI%c zwL%f*kw(kM<8_bNaa_-IhYF}dmPAt=%ZwKvDxe>%TQK8rV96w#TsSBo2XC$Q(PMAe zl5Bc9v|jjDmOwa$;bp&8;BauNi!h#-K_>iwkg~I~?MXGE_5@;5@>&IIQE|OJPrGJ; z&v0lNt__yh=7Kj9J`wcl8D2~y3Fg*v&r8fs6*_~G54Psj?qjs^>)ZW2IhB^Fj}HPF zGFSK6^9}(GcQCb4U~{`e-h~ZMI-U^Q{bhI8>LegKzKs z(pOq4s83v8n{^T9#|J+EDA zKI-lgCg&uvA6Gh}q|!Y`dto7gl;Byr>W@=hkjCk}gmxI`=kjbfA60nyE5dtvz%SoG z4Z>!{hzx9-XlSh1j&bsoQTGIX>F>;yJFnw))n1x$DzyYusrIcqQ*NlAZFKXV_e9(J z_^^Tm$2`GZREQV7F6-2-mkD-ctdA!H)7{m3wS)!3VZ7g79&V7Vy!9{Fv1ZBgW&k}? z(pSY+4jfhRkas|&k2SZnurW;DiWjR%fZa26E}!w1@g`{rTyuvDSKk3wW)UJQaM`v@ z6ds?A#@Ht`J67&8&2lR%hD7Pt-;XtK~Z`XWr>) zWR0GWi?6?Lm_Dy><5evQ{%(z88zlWP^UG>ev2Akl0MVE6$BMNt-@KYDPj+n9kWYxb zaf$DQutd1GT9h)V&9dJ66`bv{Y~TN6F|mB};`~v_`cJZ${-E{nzj2y?e*+rN+~3W# zfnR`a3io&8>wm^~V)^DO`eTWI?v?#|iEO|o04U7@RVILx!vfebf13gax)(qb;2Z1Z z?{+U(n15bh{`IHml>ES{-1T+ z-x@?Ltbhauz}5LSkqb=T-b$fz0JNsRALPF*HGi`J`lWCIfqF((pw9d|U=E-n2^av~ z8StNsJpa6xe_@N7nGsOa2V9B(Rvf_kU;{ixfVp&_d}N1vi`@R#hfQnu5(=IEmvw=lEYOa>hR^8ar+ee z+e*Wo$CoiD{$%P8?Mu!60 zk@`&pZ2Hv0c0?-N*SA4qj_h38PM?Q_eo$~tB(tOT&ojj%kv)YgLS=WZq z=ki^=2D3CvQbpGw!#)~^rkS*C5@WVWr;&kum5!G@zp88F9wlp7v(~*{beD_i!2>~x zc(mt&{uObDvON@EkY(a{Dw)<&%nDakIE%{Ht`w5JJfp3XKFk@Ye6HPconubo%psw( z9!*ej+-!{K$*eV+;W9Wser)`pzO-eoyKElJm&bfMdGYPpUb8iiR^z6O1$5@~Pw-1O zNA&Uy=XR`PkrOGBNa)Nd*|D!D1ip&lP$GV5$q+R%`Lvu{io;889?p`FlP>5FdZrwB z|1o5>{BytevCM;MF^X}|WsPjfDmFhn2!kg#EtY*GpU7PCpiJx37B}C$ z*qrxD_~zn~9^}vbUmmkjHKl-Hz;|-Od)k|XMJF=8Nc&X!%bfn1`X@p}wY>YeY*}sz z#-e`hIR>JvU+B8Xk%HKsQauS*qkY!Vh5gD4;~5{*rg@X>rxf%1U7`#+H5Q5rNy>vJ z(+?e)uekg$K6)tCMW?36UA4pyz7@Enk{#Hco$|T1)S351vJt%STfy%^45E*{Jf4_b zDD*8q7bupZomr_%wZ5b)yLtFg<1*ox_K@%b)6H>%UDgd<UtjO238BzyT6hm}f8E%Um|8wgVY?c)cc8J;8pBmoRLI(E5*&uv>>3$onRg@L{G3Imd9_ zxD^cB`%|E!N}v@&9HKEJSDu1UOh?syj%RYb4iy@`TnkiKnl89~&vDqQNLn_PDk|4Y zYZ(W)$!)IvZqlY3c{7MONBC;`z&}GBbMSE3!l6?@*3P@%B=MKwlLQ4aB_LBBps>3b z%92f}fbi^$SWPHc1CCxOgD(qMi|ak$O=cy}Ad3%Pz6*vJZw)Ybl<_$s|zjb-&%KQL01fFEB-~+QJfyj4~e%~3ub18cO>ebEe z*R>EhS+Jp3&fJS?#Q>io3$E@}Vnb(aJVVLx{Y<;J=r0)EIK+jXftK>vA)#rv-A?qL#mc#@R22m^tQ4}0))oBsIBGzF@$4}Ivb$`W$Wl%TI(j+vv@stYx zc4u@Jr-;yixtDXIQ!egGE~49246RC$Y%uM-?8T#GuMy>Qn9>%ub*Xokr(BK6vaoa-iH0?qX&}LP&fl;Tk4_t&~;wo`{GYLMzAr-nquBZg;Mvd2Y6}s3l}g%TuOs)`Wv3BCAGpSV$=Z*}Dxl z2AoKZ`fxBF4t2VbY*?qB9<|00Wq|jUvIAon2i@Y*A1|Hi33?a2@f zQis;Oq71#Y+J#`J975b3o(IbgNlqmPmD~yY$PeCE(xuF3=|1O_n>!ePj=Ovq`7o57 zV$OIS$Kjn`h$&6(8<&?^>v3PYrc%AtHA0@3925q#7ma+Xv5D{$HD{CiauPP-hFkk_ zFIq-2@!C_TH1$Dw^Wvm8bmK*xTKl>TXoU)?PCnvl6Y{Nwwso$Qn* z`|6=7W_1p8)!W_T@BgGZG3v zFY^~(HPG*7FCW=zMd{5UO@YTvfvb`qm3}epc>ct10+KaO?v1~dq@WZ0dL}BTU!hu& zF9M9!ioDAn2gU3%v?@PAkF?VwqRxZFU4F)z!OpB8VT#VznM?JEZKk^y__B<+EKRBmrNEyrXB5J|26Q}GC|=N@*od3B1y5grNjW+JBUE4{gol3S37VB5Ip zt*Hlb?b!W7B>x!3WiCoa(EYs)%=(azzzrdPq+oX6nk4(zQ4Ct4U{~{4A|HMtYv>IE zmmQ88SLYFg$UtS-MQSfr@yCK6qYfHHWSv9{O+vSCM9iGLeM}qE$@kJONeOG)Sde_I z9p?aZk@WA2o@yS>t>sW0*Oh#%9>L!?Eq6TEvH49KTJ+#!`M7(JJENaC>pcu# z(-GI+Twn9b-9^BH2_k!aR6!PM_vRt#OGv9voN!Ymg_VS&U;!V8C%P$v2IQ;G%XMi? z9VsoL6W8%3KiTj5hmmJwhh9%^V}Ot@)H%4(I%$^SvzegMW!^18Wf*WPmC)T@3O znPAEyim&(8co3hDc$?KA!3bKd>*rgb+Si+Ciu>QY8gXshv>+`nA*e#&WvZwS!aywOVlD7 z9B4SE6o#W^*jrmTD5y49LsNBY>Ee;CR2HstpFuH1EFdjV$}Sz$&S~~j@J7_8zBQ3S z1h;`OzxTlrX7N|$0)-7u9qw321F4sUG`EMY;*@SzD5Q%YMB`yxBTCWV3rjN= zmVV*nsdHPltqF^Ch)$f%*o&LfXsq$I>Fp-GN1|=_f(6U5DckGXsrg zAq?gYRfP?Y3Oa+IK)6IExbgA`TeqC!se{A8`d$&{7EP(7+NPskz+A{ou2CZ=$hIQs z(1pL{R}MH-+<7z_hr)^T6_)rS#^o={CO|dgU;Ll{WZC2o!(VqPe=w;l7+3)0nQzXM zQr5R5%G=NA$H14%?Z=C;jnVCQ8Q|yjhh_ZvbMlw?FWnkbfPg_cfGP-Z$pPLKz<|Fs zP6Po}k~@5+1OMY+(f&`s1g6t&otA#UWCP|1Za;wUi1O#u#_w}=zrrQz)?x<)P}hFP z3YdL>fO(6%g8g}U=~sfi{W=HOYrlg97+90Gi;fY}((ieEhtw^lJhCRtE#Tnt^ltmMF~*lpcY>uv@*UzYX?JhTeaEyM8TL zfMj`F&i#%S;I;vKgYE8c|B}%7t!DS@y`ew=J(wLZeE)&@mea}&OyJ%%#DAD$}46-JK5%37S8a2Gafoc5NO=G3( z-6Bd1y29j>{cv%2ug}BLUl#@=$CJKPrzeD=jrLs4N4JypV7#wmDLk9o<5_B*n{Bq- zze!H6K4mQ~YHV*#7*!gbZ=9N7w{lfembfq#o_V=Wx3sZN$E4p2!)Y!=DELle0NN3i zqp`X;A~#=}b5rj56xY7%!5a~*6=`+l3#t5N5UUhdziIo4eq0o{DLQOHBA}|(9bVMc zgXJWv#RTJcdPoVj<~6!|`pTTAAwof>B-7OgU9cevL+&G?)PqpCsrw-|kg;_cG#IS> zVCx>l9)pHFghw`rPVgE1dIITqFV$p7)g2rR#e1Z6YAc?0Jx;3!=><53wg=&%2Bpq& zlhc{RBD*3N=VUoGm!^du4S}AdnWQ~aYdop7kDl9Ujo`7Mn@&<$FYL+O(;YsG7ON?n z9@Z%xLxLOcbKticLF7tLQisgr9x5X~LN^YFqgN8!PD?#+>lke4BVH|{q^9sl7d|fC zi@7YNr+3N;g!jqYA6{r^0M|=jD65>c9&`SL@Uid9kW6A|($>)1CFLToXEd43HM@Z= z<-)vvf$tLGSC-9G!YCd+Dm~TEX(TDQdXaNAYeD2zi(22K@`$r>)#PCG9>ZYsf@;m% z#xsm_ob8r^k44u*HwC+^qh)T%CWn>hlwIyEMC*zPollOtVviI37R!rINU{iFEnWT?;;^+!y1A4aCq*lF$;w;DMHD=)6;Q%uJrzUmfS2WZs+)s{8Rk zX?S9%y~7NOOQ~k({wPHG)nsY|p&$gK38&P()6>y%XuM;V7{cnAQR|b_$z%a$tw$+ zKmpq0mR!e;Z7p6iv+AhsBT@C`0-F+m#R~Hqq#++wSgU=YPEYu8M!o!4Og@##VTbqm z_$H{Fzqd8z3^Yn@Pi=3YvpSK|DV3$`)?q4{!mqG;f*_gFqPQwwEXDrneh&*Kcj1X& zO}By@p5rq@#yDMkLhM3Z*i5PW%K|(r2(-w;hS|qtK2D5M{0D|qqy$FH4soh1RGle8 zMT@00Ipqx4@-2P+g%w{SC1=YeA$+7(+p(B)pD-O%^Q|qc?OAX3di2c5KPZ< z?>KH5z(0G1Z>5Kil1Y#ob>_V{CD%XshQ$Vj?65MDF!E;mLoDF~oq+ji5)=C+i2=j- z9L_7}2G5qvxU6jZ!pu5iN-_7{ocsX?fy$W=(>4#}SGOH0!dfr;XbuL?AcJ1_?;8-f zD6u`+(ay@(I`N~AYI7fe77-Iwqkay~8W5ALu{Gy}Nwj|;U1%*O9veP`j zkyqaD__35fsUGYt+RzeMt-H`BdK4OqV2@FvLQ{+>Gc05`QLmrc1IV$hGwyD;Bh>>e z=V)!-8!7rT4V`Ig^~4}68Vn4j300zljz?N?xTo#Rn(t@)u&asGG2Q2?As}p@m<06< zw%B-Oc^70A&sv>5Q+dPB1;aG`u3DkqRJ-J31$$W_cLmyvUgI9Os~0NBM&T&iIitOG z7Bn5JZWig`0?CiTm^(AUa(v#0YaS<6?v$rSZII8vM`2eFv+IW04r<2(cS-zonI$~B zikoe9*sBDaOvm+E53+O;R=dJGl_R4vO}lYiFS)vPgr-GXB{}mcG5y(>jrvppSEP_} zh-7PK^mFtR*ORlNr=mW4Dmtc(tKHG&X@wQ#xf5W7F;n(>@Q!4!WAu4t)aq_Jh}^xN zXfA%(-Pxwof0S4RzC>lge3`KSdf zE7y$tW#lg%9fno5g+Ed4WS%p;=`wjy2G&IAfyl+(7D@y;^iuD#yd@_#Fx4O)pn8?m zxITLlEg?_uPqe126)S3nRIjsdyOT6JG&ea|SxK(S5sot0+&TTg03?xZ{-y@fAiCJ? zkjYm}?*)WZ{v>u%c$QMl!P|!87m>jZF9SqR;#^i*TLMB3Qz|{&2Xc@@WOEZvUKLP! z+gOA-pH{%UeE?sFwkBnn`8F;pe@^?62{c^%CX^@c>I=T%3=xk`cb!Dqz^r!DV4Jhu z&t8>C)eo}*A1%q+LCIvVx6yK9vbA&O>E#J)7(pZ0JXS>1-Qcv)8l%LWUR&ITmO~%6 zn{oA%>dnlXf7QG_m5ix-Zzj~X`aYbmcGRqkqpu(ilh88pd>N@F+puTGbbhQ77-83A zXv3S2GHzP*3l5j9g|dQIBZ8OR(6~++$`LwsquYq;X8JqJY_ZC&WpCHR9vu#W1&7NhCdk%m}w54k>>ottRoJfeev za7cutq01zZPwG*|<@y4%?Ao##XP4hOYlgG@F*Fmfb#rNiGllB`Hx-CnlO4y4^H(?= zl!^n5=wnuUfHi-nkv!R41u?ba2*(szFA1#A9xIfB^=7YK{JhgZT0W;Z2$LynRA^pC zJ(_ktD>2M1`aYBiS~xUg+nazuiISPsH%rfhU$h!#;YYP$Ms_@+3qyw`Y2c>N|n5{S%5m}+d;vO!g@ zes2bAg)9*98qcqX-Gima)7ZwY+OdUgNs%qEo4pwYZ`xjO?;W>#Rihog&?o28Zk@-M zJ?V&3eT@Ns;qxp_wL<@XN{p0du3-c{I*tacf?NDj?v=A4_f(79yWNvvTO ze!VzB_%a$VTkHlJi<7-}k1;u-Hpam}Q)$mJ6Y|=Q!5SrU^zk0rL19#2`=ozKTnhxd zH&#?>)tz~bJzXOe3*q5bd*T9as~{9fTyMc4ppQe&bTm7D?!EZYwM7z{%H@-V&x}ag z{*>5$UjN$}nzOo8GnBRr_2UnQqVdty<31~-*O~hYOnO7i-5nqw3M~F`OA>6?WW5kD zWHtk^9=-)-m4)-rhw?EhSK6s-ZX{+*-VzU1ts~Zs7Qqh4{X11wh+a?&+w;jo1)I~dC?`Mf)jFi@g4E%Si>#Cp<0dqQ<1Kyn5>1qp(l32z2- z`Joo-Jq$?Hk<8V!y2Y=b%WM|jbKx47e0)K?FNJbuP3ZZ;ws7GXA99+%{7j3x`o+7= z9LcAv;F{B>?~U@y6CbVA<{9c`5+wL#_uHu_Bp9z3B$ybke8!dt?;N8DoK?yKPn+9G zfs}rgD2Y9E{8e8X_7Q}g=8APWHDYghr_wadFp79xb`TX#qh_A&gRl6LCJd+q!~0C) z83gB>6T$Xf`y>0!F*l`y(r@aV13{r0nbf>fMuH)AS;j~euky*0kl$kn@+3l%JqJMv zP#r=*YUnIiS-pKmFChxPU+zeT9*sO<2e<^K&G=PR#Q#eRCNu! zhKCspr@QjHad^24AQ|9NdNP|SR;c`|79LT4Qhwcwr`)Eb42LbxC53ey9?WJ} z2_Jd$@_2z->5CsZ*MKq{UubkxodFm9YbF&Nrv5ux@{qZN3l*~FQUk=k*&v5vXRs{4Se zR@4bKIwWq7VaC8U!K2GwuFXWXS;&jayoS|*j2vlmesw^73~nI*1T3x&BQcB0eG*P1 z#G!mPRcFU=v(X)^V_y5I-a9qu43trvPv~^wO*SWDh*?aITPg!AlayO3dY(r(SQ`~r z+MByQLNL&uv$n%NyBT@VB@J$8Ksloh@|RFRP08kypIQ>hm64C_Ve<}hd{~{e;NqRW z$WIkl9>7R9YjUJSH6z)pqdx}`=ZcY!W7GJ{P`G@+7`G%ZO0Z4eCy-?8MEcd}stk^^avz|WWQ2Yw2IDrNMkba?Wjwn@+3m^}3pmNPU%`J<^N9k7+W)>S6JlU`E~ z;2O9SdeSARLt{AQ7|^i>ipiL}3k1m?S;o*lW$%+13lF5tX7BTbG)nXJw31?v2orzb zZlKL7D|r(<-5OmAXK{&MXRISYjg?P|{3Wka0}V^77SS24HK@jthr@(J7A0Db{wR@P zr=gkaIKXvfmp~y!)-IUkbLK!1laY>HP-|dDKK7w>%cioR)7#wGL*}4PibOW+H`fE0 z=&HH+7|6}UZ%d7!P%0TLST5gPEou9aJ6W8LDzN)?bkCH{qCRXUgU`iwn|Wy^ssG`m z_{I!tfAb1HMA>alB;7&s#dgLs)J+_%nIc12cf6!~xFv4~PK+IMnYLw@^al~is>B?L z;&I`2(B`t!l1sR+997O|ZDe_~8=YQoUJUt`Wege+8AL4VxTqn%A*>B@(Ofo=QSTSW8>vlb$ z##=Ex>q6ZYUZ^1mq?Ot0j1%XugtwBDT9s~P(BduTs!MkyO9S^UPCJqy4d2+0pv-mE z=<}`Z6iwa%3+A@!Bz(*>qqY8}cRiT-HGw(iiqbF~<%j9{eh56S z_^?Qw%mRbN^-N|!VMaEv*<#&u0e0$@=w}EFgw4>3UZ)KC;Xrcu5 z63Jv`X-g+CJF^lVFI*>nrPVRec#hPQfro)o6$N7@3e<8$HihZ}G|y#-=Db`Z(sL>< z9If6IB$*)H51DW|<2rhF;JLe1_34a1jm|X$(uoa~0J_Bf^To)0h5Fd7$+VbRr^zoOjvPMSY>+n|!6;*IhR&Ooe z$4d;0wdlw#R0UL($)MdAxN1K^#`k>dPh>ST@v>BHu1Qo*&owUNNlgfSQ@w5DM3?q< zO(QJfWr={hdrbjh)MD(!blN=4=W8(>*lFiedEj!V%z9aV^o(C&l@IN zEL1>@lBb%&Bo5bG6E*$ZnvUXi+xTvWg@b&Wi#0b5XYyh?@0(3j_BhRLnYJlLT4&x5 zBO}(zz`Dja5LDck7t{|xlE|4(CL8*4@@eoZcyCqM>|Zpa0FS4CwMYI-%_xAG`v)b$ z{{U+j(6{^7qW)Ym{obIkwR1AJHuw1VoKZjkWfX93BIUfDUFQTk6WoAF&36+mK)ve@ zRrLR}`}1q?0<8%yposnhFAKob1vKBjDU96>@6Q`Rzt)@{(E0kx0nvR8(6<|Sge)ksx-9?E3 zun7Fw*c;#rCgR<{&_9gBfV|u;1!L0RUNmUMm*>AJAO{ zq)qN{H~~)KzjQ^fdI|i-;n~1H7+1K-94s1&uG7N*Rp{C!y$kX0Q6cJ zSpkI)Ho%{Z19)ZFK!CsqAgKAb8^b`~^S`od*#Hx#TXM*EyzI=k!k=9K0PoLJnw9x4 z)ZYb8Fh*8@4)8nR?^0`b5Ae@3+AjtC8%qFif(1@5fFA(Np0I!b^nfqr9Sm$)fS&Tt zw!<$)3+VH(12?Z9Z~^^PAWPh8G~E#|U@rADygzhcIqU!jM2COl7`v54V*!HrJ6LWY zo8LOW-5Ko9Q`)Zs3p{xafJ^lQEDNx2xd1KkyMz79boOhn3T(hj#tG0vSUG{K!tEd( zKpo=(*l@StS%B-~-#*2Fr1GRXu`IOyLhP5m2Nf&Erk4q%P3vH|L& zfV%4s@(?ih1>AG*fcNJK?U&;HekgoXSth-`=m69qR<>I&z&mJ70U7+i5?l~)1qVdh zNLhb_3%XUz1@1X_$NTe?X6vGFWUdeF2W~_Ce`goS#`fzusKhI{r z4C*Z->sE;Px2OVA&%pWcPoVxhkJ;!Oo7~0Wk4^bMpsaR%T#g9Qb)>yg$!tzZ5T^HVG6I01of>9l;9tX|vw`zBAxI zCbs{LQ3xFLpg)b+)@Uv`W_P3cPU}`g=SxSt($^zZvsdxrzoJ{Gkf(TAh|5S8-FFLcXGw)>?mm-6Dh(ojD5*#^$8 zZ_#~7R}9P1>50yz2+OL&ub~9?)zX_veFlnlo%0QClSz&&=2jz4XG>`-OPjD45!ff@ zi{{o(V$@K?;>UX`IW)g|5arEY70hlReWr;B(e4>w6J}Gs&?!%e=WCbq*;{ zj;MKQIB?ER=CmR@`+a+>7<(2%?w|5c4Ud%7Pd#@PNsV$M_L^^VpVD|%G&(}(`Il<}BPWSo0cHb5w=+NypZ0&xzHIA*fPK_;4g-uzgrn)|c)@dho4S4AG` z3k;~|O!1i*iN*A3Zp>&K(}s&KP2@a8D z@dKpM>o4jF1MoeC#Z^6-9<9?)MN#HZoMDSqaa#^A(^_7>!@ze7(5ZV?zuCXI%*(*n zNIvL-ONeq9R1%r}6+r!y*p9gPcRTg;eK^)xBGv2)E z%rTp!Y9Jj8BMGv?Xt5q(l@f=VSFQ=k+*Wwd&WRG-lzQS4Jxs7KhYr~%86 z+AW7JFW%2-FPh~;==yg?o7B}dQ_@vxtc;pi(8AOoS=p@vhI&a##LrR z2uR!^TZY8yMt<>u&A`s}d6CV7Z(PMj;o(64O?QrjO_$PKp+VX%xs9%c*SoVj?d#*2~(rW=WJ+8I4?qCl}Q zP17=zZ&8LrrR_%<@(o9wc#Oge>({6~>c7G_max_DZDS=OJ%e#I7VV6a?&yGTZ{w$0 zufG&gzx?tA9*U4~Kl*je9xxl74KXzpDUDl($C2LE(ZV?MZjprk;d=T;8NWDum=L3| zz~ldqx3>(aYTdqv>5xw8lI~4xI;4^A?(S{`DM3o(X&=*R|@2hJNBZ*_O<+|68bXU+3xQPsZ(OSn&R3Z_1h zjF3BD8Y4kPo%A{_QH>+2NE(|Y%6q-i)klm_3omD>*hw}uNH%lMkG^dZt42S2ore1jYrLt1e0rCeZAkdaGHAu$^|9_MAF>D97kSaWSP=~ zJl)g5@5^-`#4dYbqGjKt7}J=w^E@qc=kCjxK{6hXUZ>)PY@X-2`|Fckb?7XctPWe;Y!uhPs!8O7aou zon)R?ymM%;UY@W8ar3HfR{I zu1MyQeRbq|7uAKFv)fgvCq?2Z4Nc=KV0Hdq+s8*!QAt1L-sSex3mi?EwESHC)|NY_#cvaABHu=Z zyBB1?eeC%%TPp@T!kenG=*$+4^PYuGhSY8K;({lURL>tFKT@8JQQ~}1!kH>Z*`dJe z>3Ljl+-TA_@777HnudygfhO0qo_4xL^#-ay{IK!C0nAa1YN7=Ea~?vI)>(O$hez8e z&vo??g7V0RKuG=vP+_)r4+3o|*O3ufI#6;j?(Q_Uj%GiqDcS8* zaeB}=3CtXR^}-66we~X;ZW%nb$P1|Kpiefn)yV84clqD+(BPB4hvr3pFmHqL*e&I0 z^)<6h>Pah1K1NHBD(=dtiaMjqgaal;W~hmky2XQbTe194W=o!X23dK+(JwBGnZdXt zr$!0_Cd=y5ZL=M+F3aS5G^|MCNAVl7Goh+4RMuQVQ9}bliaRN9pbot`7P}_W zU)mkG-<=O-bZ7G1&yJN$^8B(b`ijwALQgFavR>=iu4N*+U@1W$BdIhD~vGvv4+I{e-8sMxC1w~?ayV*tKE@@#u(0{S>Dg|ZP z&Fi!p>;~1Wc)ew;HsB%eA`2aM&kY(0BA1fi8qbTcy6d&VM2ylCT|~6R*HLa#&3G;h z3(m+`pXQJBEwS$k*6y#4Z;u~L>HTod*GQ~PFm0|~4yFPTrAYXSxS&c;quI(|!EjYp z=uSS>8#dh&1+_K#gqIiLV1zpDFy3*tH&9XRbIyvc^VDFLedDIdkzyu?8KL2jO?s>< z=SU>6v{Es`Qr4+%ehmH8>n9@q#$t~%wZ5Y)|qPyJ39@~?!9Ue3~g;)2U z4LmjoQ*2owHhYFUJ;veAmzbtq7{yey61z5vL5I(C8)k0EH=_Uq*iopA?tEd$E0H6{ z)W#qfKDNINrVu|8>h&HNM|l--1_MLn0K+c9FBBg(0-DFcGwC>Lcf5FI{j`GR0aJV{4iZ%l325kX1*dVh++5@OD&2HM#fuL$*2%XEf)N&5$DzkxR|se8vSCh?`C=2vaSg| zCZ42R*gd+XKt^)HXRFqQ51o<|;g1Zd=k0t)PQ{JJ`|3_!?kp43bx8)$e~FOfIgLaX@wy6xMbzRYRNXHOER+Oo>6*U@B)I*q*q2ftYI}ip zZ|e#PEmlKDUq7E9{XzYO51DNMo^vQGu7PpY&6a^m?d0Hl7q`o1Cb#g_!MttwHJcr__cI;;)qXo(bdbhRb$ zO};Wbx;z$PvHns5|EX&iDJFylx>3c##*Y)vSpJkQjzu)J@I!N7DmfGj|17ofsRVb` zCEJUSTa1}+)sTHZ2Ui^CC~MOR;%y8+Cu%g>i?OI-VgTur$78#E$aZO4?_Qd@fqFMy z{P7z?B!Hg(iw*u?3XuRloe!XY{}_q-w;uU_lO_GlK9Eg{3vdSGF&cl6^M*eG> z$}h<|K)^6F9-u}8aByEsb^w74NKZJqAtr1;QStx=qWu-e|HKRdMIerUVTKO$brKc_ z;sR!?Ru)CE6qt9#G-AxCHjn1)qkS;~MWaGhSw=#|z){HIeY<0RPWx%U?vpwBwj((rC+OF?gA_*!jlQ;>%|C z4t5nz#9!Dw*0^~d*j!8{H!vL7_JtE&eLcu&q%M#xmhI8KM)Lh-*U#N@gE%+Cu{XV6 zyfZA$3SOMro@zTPdHD3CA)W*T2-WE+s^(w0H_4ZkwZ5pe;}p;+rFEPQWC;|~=s#I} zP#oouz*P_{A?BWbd`j(SzEoWGcD=n>e!nqLD9K=12cJVuvd`6V^908x&6S}mt=&sM zTno06Qwry!>!@|M;0W1BgxD=yRS;#|8y0NwQpK?1)zqd$jyc@8?xb3+S)>&~3+Bwd zp%W5W&DAGPRBbC%^9D4OFmAt3EwHkdU zDAtu++5s&%`k<0qmIh06pj{+^NsWaw@F{ikomtN=T)ERs;qVOk(!{z?!z8rh+V`jf z$36;%7}#ErXl}?GTFktGb9zm!^d-OpBe8@kL#7)VbyD3dE0koU0>7w`wTbt-nx{xj zafK^FM(5CS_v4AN!kbNZ|M&aLWk!aQ?{1R;{ZJ#dN29vkHRkG4eb}7IhT3HvV-=5J zR>xP_7GxXu(Ph~7T&KJHMx~x8b;O)UC>(Z%g;fn(&Wd=(WvPrh+nu#F zoFc2mmqZTXiQ{peb=BJ|;=f)wH{RAQ(%Ce(Gz^zWk0YUpZk;`(a>uH$su%BTfPF|$ zmKAG>viOR@q%H%smk7*2Z;veq=K+3~k(K~!Gj>o*(a3Be(VYg%vck_|)^320>-PrZ z0Mv*l`$8{b%`68<;C5mLWVUlMTw^D%Z zWw@GAUntS5&HLx=Py5Fguh-h2m9;arI+^KV@egi};Y$h)i%f5K-r=^RLXt1jDrjD| ze3Jle?DC0LiJ_c&Rf8=i`kmWeCq=fvI7`x_6WGUq?R2QdR5!#3vPDy@h|9`C*qCUfxi3 z2D8*W@A68jy8>kv@;Z-CPR>H{-eWcm*9BhbI)-$I29~x+&+sns29%o!sWq#PL3zDS z#3(UnP*sukNpt37Nd_)A>LD{JYYF5EDjB%_=6hcViF1Y{!UoiBAAa_IReV@aA$!MAy%vChbE}i~WWK zX+#1Aqo3H-%qNDo>p}6f>OAS|WZy-?M;lCL&_FG;INR((@8QPmr$Du&a14?i6OrBj zXjbEc7`X}0HogVNR^ZkmzA~CgNw_6ADAe^ur-M>$aKQF#H>Sj!63STVLB91{vTe*DcImhHu&JVm^?DIB=ixF)!9usBMEzOLjl{63+cG1{ z{SDuMGR&A!su|+UwHv(`GFj%!pYzbozTnfMML|S5*rGS2ke$gS%}GgpZ7DyzE~b6f zwk@TQ=8p{HmM#4PB}`5zs+cV+DjE(qB~#>1K7$#T^^}$isu*i5LLGGY?P)}qWAYD6 zZbG3{r9)R0=)rGyDQX8?CAwXAlQ&KhWyzR%>_5ADF-O+uUu?_4J#`s|KBkyZ5Q{Ey zLY-HVCNAMY*SUBMW)(RgqfCB$C3x$OwD5Z|_4<$X>vt2-QCoNXuhdD}BfGEbsn>ds zoyEj5#B0lvmok;kou{$%PYBRnW5(zQ<5%YA(u_|>J4flfgSSL3rJgDzlB&9RY*9CY z0l&ZE-W3}Kx47|;CP^i|S*9rM)Xn8Jd;xt9GZFMipg{E6E*pXil*mmK)L1mIMM3) zCnZj@-w&8R>#r3d%}A}#2PZd2M7k=*M0_!NdhdzeF!ow4vrDI&i7AZbNfcT5L>MlP z@#km$kA3n|7mZ!I$5ka?m17y$bF6S6x*e6%!p^E=+Cx<)p7q{E7LBRko%5)SU_oyW zK(241zvg0S)O*0d&(J#dcALEHaLh1({=jJB@#j)csWjU6!)9Ru%eePvz!%xvqZbQA z2dgS3w6wA&frq;nZ#7xBajDY$uGh>Ossfl^7xy9SP5O3GBiLIwG6kf$;T5tPGz~1u zo5!y6ethmW{ptKMTX6+Vk8PhbSn4gDEU2<}e%V&<eabsvEXj8Enp%Z{73D1xqRm%v_1=s>Jz7ae z3i42_(+@N6fx-{R!R4VT=f+wOZLo1LjeA{;rDU9iyRRvu;8{fcuroDN!ShJIlL!gQ z9)(hmcRuMHK9$;7j{V(x&{L{A+({s`X8Eve668#eMWvzwH&U;|a=P}O`i9I5^R_k% zZBJuQ%{`lmHBj~^ZO2G8E9QKe7V+l0JR9D){(0YaA_cmg$31 zh##QlMEcp**SI8w`tFrE-Sy~;;0OmJ{9GI;5%HtAMsakzCf zT2J5;L`KbpBc0ff@={s)brF$6;RaKY2EFRmy2n{m5_-_N-(pgsi9IylvDKp=1c9o= z4<+!A#Hp2vXmK90Qb%lvF~SG4N;%S#8=u;eC1RMyt$4c;t(dWj-;0U1i_(v#(1m^{ z%ZGdaSj%l;+3f1Uto3NjtDKH&_oeQ^=Xcu7mg30QvnJ3~8935l%*3WVL2)BK64bMq zo%na@LCKmo#D%U4qzHWpYV{u?v+H1V_TO=AUcu7!r|N)HA?k7!sQpfs)?C^mtwt-K} z!s$@3j{|12E)X$_EmaY3#~_QhvDOHLACrWtKNf4~W#dJtCFBj2uYO{WL$FhbV6`KZ z7D#Gz$LH#!R@Dgedr_FPa5Xd;A~C4LCs2k^5316q4p~N^mU`cCClMg(IgvuaF{V{G z$k{4OTRF{3`L-caiq=U=-8)!7DGUM?Ke>g}RO7W0cdcpNDvXiHavS_Qr|+JPtZIp>Y+ThkK{M2k<{mkh*<`q3S+BuV-$HjD zR-CkFZi*VyL5%l*>d!r-(KON1ny=<}PORPAji=7l{C+T>zDE_VfIUm4$inx4Q|nW- zF8m#{e(INR7)Jf`Id^7@{qu56+U4X#BSsuq=HE!Hdt=Tfc(ZvETnh^F(l97Ev-8oo zT7~sd!j#`73W6Ch9k@=~3G;(nrX?~(8y-zNuB-Pt&+wBisPWR!!!=?JHu_DcXO)KcOx?K%Z>o6uI+M8+ z2vsO$i=T`n%z`^G@6_!NVO1~og&QqV%qH9tf*UQ~Dz@p|n?YI9fBb;3MIIhnb zk(zmby#~V-0W_@f>?!(eo2kf#mPxr4jD|77c_@`Aaml)mT5OQ@-8aQ$#Ul87*DXtJ zbv(sJ_g#^SF^nmP5S+zQ&|~Am+USV(p*g-RpVt!)cfTu+6(c##u@^r*+2?J~J%=FT{ag!Z-lo@~~uKmm?ZnqqDKRUdg+ zUHNxfu!hdH(XlRCXQz^Jh0|tUHAGk|naoiEf*K9on6ydj988aeSJk*liwzHh#n&|h zgZ;<)Lg}>2+y-;8V~^F(gyqyq=9QC*ULME3x547{|r6 zk$ov?AbHegmIH@58F9X>w`ez-W(qYJQPuCo@|~&c3itOPZicLddN7?&g{)DhHarm2 zsKpyUypz&p7xB2Yvca*1-LjTql=Zo63^fECgbtT)&0hg|r+-E4E){DU}nNh84 zVcy6SnQyv2zN}%3#lsIB*!QvsV&GtynGlwelTI{kZPczPRZN;HSX-WJ{K8WwE!gud zO7!nS!~Z2E`u~ui5Sc$jZv`Pb|DR6Hf1yNwodgBk-~J-+{z-y<)y@4e<$=@0%*+B9 zQ4T1q9j$&rVa*9>5xIdz7BHj(2claCx|_g2Cj%JJRPwU_gd`kj<@)~%GGKTK8RGjV zWQcnwCjkBDA%9<;_>V~?K+hYCy@jWZiMzFlBL}C+FT@#eXa)G&zv9dV#22u@{4YR% zU!wT?K?4$YFt7;zj5Id~utj&oq>io8*kRTKNQ$+_iBQGPAm`@ zSPejL1L&&?7~so6K<_38AioFx{==j``0s_5zcBum!ULo;j|*u1;s#n-ApJoQHXqX0 z00w$DArr}e9OO5vBY$gsc=^zB!+<1RxObyUn z@_(ELU~@}*6F2K0dh_4uBn8^cA-y3GM`2E2^f=Jl0(f$Af`I|*fY$ZLL;CkZ$v@xw z-_P1#hspuyeIW+Je_{nXoVb8~o1e$}_ssXNSbzKG=img^0K``sP}>9Z-GSNZ5G!0@ zb~0n@8gx0^cG{+`>^ik zR88DL)Slc&aRS?$^VGn?DL&~zxiXo>5HcT+AT%_OOP@cU-E!ATpyDem1O|V{(;ZJW zJ*(vRc$R((jFDk{x|PxKx!o`DbOu9E9?X0+O<|F7c}cv97}T?9+zobFL5U?N{P@U6 z&q3H*WV!gwbdw&d+Iut1Bk{$wj2=Jw3;~T~HuqUD;w8!W_L^*xq+-f-H;bI0!^gn< zyt5L;S%=A2EDiLwA4fehcKF$L5nG(>$MwnI90++hJq|C?+r&!^A)7$eo-|OwUm9jH z8##4qqLO)WDHXhVk#ZS0FS;L^6{?VLje~NCGAKLusYp{Jj<2Q*6UXbLGG2pFqM$~q;5nJQ^rhFcVb52UECJJ&n^ggrCf2XWB4l(MS~ZYjEu29HefNheR!05?k~>d?K#Z68_i|3(;^^SbnS(qqMg#9JvWA6i4QsvQ2$_d5xt#S!t{4HT6^S>K^2bo&f9!g+$u+ zXjSZs{$dBYCEchbLlm7@njZh!x0J?AI@83ghF;-Rdjmp+(yd2(H64=sJITJO8&Brs zN8X9$w3IeIJB%!!yBz9Kv)cB%{~(EtvG;mm?nQm;8=~6$dm}FvY?7w&cG{TBh0+wcKgXvPoEpun*7)Ry zU+T^q+!MTay+rj)u(%xzjFyfq3Fi)w&^r^-dxCfFG zg;|(}r8DavM9GSuP-Isgyv(0A)S-%n=5=k!&p%8z)K{2_U6-)#)^=%WMTFMtWW~Ay zz4s9kcX`dVqzP4T62bWUxG4 zOW%eht-rkhM=j~kFB5$tiFt7h!%~+u_PA(cGr2em<-ln;d{f!IHIk^VsG$|_?(C?X z91oZ_BYao6-F~rh{=Mvhx8{z0lR?cGJ61cAzPX0Sq~-vEIkwMx!)RP~j0J~t>u^q` za{bMrw@#tM1^;b@Y4N-(|2sPds#AG;p1KQY5UD|KXPSg=KDfox>wl z5%do$SP!j(v`8Fr-fLUEpqmqP7NsNM6?oFFBDWAx>vWOAG)CPfTad1Y1djKMCM*tNo~d>7;&$JkDaq&8yw8>X zG0I2t&2&}m2Px~57-5pjs)s=tt%1!?0>=tOn?fXl9SVg}-Z6#iHk^ij;K=r@ixj=^ zDUW(Km^A9Yj;XAyZ5zZR$xY>^vRm3Hq`Basb~df4;5JP0&> z#}w2%S;;S0@;i)OyW55x=n2_r-%txMF(_b5L}&`vjeKD*4>qyi*<1HB~& z98VEh9~3nrl}2IlqVfuI?RGyX6MTA6G?v*pgH|KF--{BxfOqSnm&{ph@1o`Tr|C!D zaAdWyKF%4NaeY%=$aahN@NLmFp=}*gK5NZrAy6iZ-tX_?HOyzA(C*<<(8w6?LtQGN z3qj3v=o9tBdEhTZ#vxVvD66We7@M(P7wqk7d-gh)Y0`t0yzI!9t#L+;Ex!P z#~5=R3T;|Lt5x^~zV|&KnQ?FWsN7(*?AXl(hVHt!uJtPr_En3B{jtxkh&zUI_ z@``j5K&;3FaSxWJCAA}Ix!SCA)m&iz0u{SKRut-k&0j zd}#g(>M6m4lh>J(18V8{OZkZDCXT+^%ffqWCRTFto5xEZ)-oj)ScB*5m)Vj)6cki9 z-bz#1Z3-5*FhAyXskJ)Ywpd=!4kz&%4o!N~IFob1w4-Vb%=3}4kTG@^5Y&l(-wnRi zU(_+EJQCw9nJb(B>UE?sOCho6+aA74tdwGdYMDb#rmmzJs^XirTHQ{YP`R=&FZk%u z;`aOiHg8+I?JkN=tCF zwDEncd4=d1`#1xOr|jbOD%`LJzg1As*h^N?$8sT|P=#k7MsD}POQeT7NufE2+FLR1 z!k$S4DZ=}J%(z~+4|%(Mo<5k531Wr|J{e8EMqYn)bf-_7Xu5^T8jq^lJ1nnmI@0NO z*@&v|@PlyqC=^)H2R1~$c)im!xH&pVGGe%~ciZgAG;^b*A6+g*BBbFGBWteXS}QGr z_0ABSmVuLZ&w`^Of=4X z?<2>fk$i8ia@&J8;$#exP;9uQ1~onKeFM9MyQYr^u@7CBx3ZWla>lb^R{M|3_wtFL zqO@I!4Dw)Wo`z_dgvSI^M-|J_FSOq)S?Z_jkTUJ-m|~g1U}?GMb#JF#5?VnZFdk_< z?a}qpkgO>|3>?{fmVSM!#<4dl`jZos6SPu2LCVQSg88kK;`vnL5J7RKo}Am*Ok}Ql z>kT-S50O3=nqsl?$&(Yip*;dGWA&O@E$`_j;g-@NEpsn9wfMbm zF}y`rXYLs9+Unwq*T4h~>+@15{yO>rNUa0L0uLqH1K};Uw%9R63z#h8Cd|8fVAm+m zkb~y@v7LlwtwoMGrY0t?(O%LtO$U~l;{zU9L;{r8rm8L23Ay+P)ua_`S$7hkG1pJ1 zqRD!^u^!7UQYrX-jPGO!{D1XqMS5Fl#(Ex8kX31Dc+_W7?0(?JnU%7ns~?^$l3a0*C~%+^xV;f zHR4Y9vA)qe%{z8teFtv)7xuH`7QO4^!_jHY;}RmLryOV6*KH&CTsT#XyZMX(0;K5W ziSW>?s6&}DN39y8@{^#LIrlRhgv7Dg{nZ>AI283TN#}X?Y|^~!@hY@?sN|?R%LFs_ zY1CA{&*sxPOMRBK^i^EspY%u`yLZS*w!RaabZj$re6H40f`fZVpfSn#K2n#|FcY1repOAnWrvKdz z_dC~4tqCIa$Hk^Lb$$GvGqG)5b>^|xY%tiGn;wYxNmyoZYqJT^|-uRep zK}dJ4b0Ar1_kNO`gt4Sqois-r6T&c_G-$~tpEOjH0i z5)7EXKyX8b!2dYT|BK)Sj17JPCnr!l;RUQXA?!E=Cl?ps!Uq^9{5;NY61@KsPtIS! z2?B8PKyU%I9f(q&mp!x`tw+MzfbB~ zSvWYDaQ)EB|8_vRAO${v!v7QGpRP|o9Zz)?0}dj z@d71K2xP#l=*QP5P!#x|L4>#_0UnTlfdr~f+(4=1=Ry8Gh5hq<|Ggy=5C|~G0ql=~ z8G%5L4&XHd1_NaVh~*b#H~mCFznP1EGgclD(31lM_D`&U2^?_Z|2)=jCZ(@f?E!x$ z4zPuZmA%EU>?9y~JiNg6{sSl%81S6|Y&L&3rr%6TzY`}9a4Z3Km4D&{>P$er6!`tK zK>s}}sZp?LsLBGJWW`=O#=9THNf$f0jQl&u!%Xjnp?O6jcbsDj~%ie#lQXlT#_I^WGL8xe?%2B9##)9 zrUKRnkY%}efwc=%{eYqrL|efPxDv4gRhu79UO*E0#^LZ+pV^OPt#6m1xPVLwaWwq{ zBjDE!*zx`s5Wk`hFa933s# zoB)9ci?fBRC9tndfo2K79r?mwJBbU_fk8}RuA zx%vXYf!O|dOuwH1&74e4Er76stlS)U?S3Jsz`AAyZj=;&;psmx@&cZsKqoDv?#})* z*K^>+`NnDdkDJAB0}I5H3lJIp2^M013@I`ae>(m9@JAAZTOf;Is#} z+b_fxVj#;2xLf`S_s^U9&%^y@ru*G+fxH0%)Jz`#!KfIJYG zO8#Gf{AQ~A`u2YrBxL)8fxG!XK|+R;094t}NAllO+@Cwj#KR4c;X)28XICdHR}+U{ z*isyTc|Q;2cmg~L*n#h$uRZ`@=Oh5f){lP&K|V*mcUt{UxDe+`fam%rS|BZeIDp>( zKlf8;zn`%FxS4^>oXjn}e<8430ENT@2t)sb$_4po18$W+4fQ|dENhFetKgTR0(R}d z)%;JS+#JBQ0Qj8vX{6szShj#|FtA>^?QJ|QAZSf39L;`3i^&0e+CW?@IDiY}*VoSu z=v)D#TfkrA#}#c52h{Ja+20NrP>pf`vbaA1^8n_@kks(Ah`*n=e1XLPnVQ0B>tPPL z<^ICb{wH*lr?6C!XV z@PPlDF#dZeIYZ4AhtRXo&y4%6WC)v`Z>V) z)g|cL{?;bH2`liZK)(wR$3L)gZ~`-P{tKkv&sSfWJ1#R>PmGIdEvPLnh8Z?Bsz(0nC_S2fUVkd@+1K zd;R7fC@z4XgUIp!L=15F+z>PIpGN%s^!3|??p(k%mL0ez{(<=`QveL5`gzRXPhh_t z^FJs95TwTh=r{u!pdiHvNP7X$4D}Q1;+t9QFH-c!PO@+FVZgx!fd&FbcJKj}84xfU zfgNJY&J7TbKS4ACA430@X@8y$Gjp=H|3?Yk?O%B5uN?J%$4djD2WGth&jnm4Awz3` z_9HNGP=g?Q^e3v+oWOqm>!S7_TMsyZQ2f31pn_;IezEa<$*40$zLXG!G6F@`>f=fb zo6DL)f_7Sc!)ZrRE}mVax`b_$V#&>0H)p8G7Ui;rYbt0sbmv1Jj$4i)^xmJTU2nAN z=!{-QY7iYoSg7=c%u_v3xfP|-5khUss2C!_n7p>e=uJMBY{?PRjG7@(H&}Lyj?v!n zi$!E*o12}5++(-&w;3m%^J#oS%;_@%Q0c>t+c(1k#X7V5`X<9Yk$vXzBq9}= zP$V+=&@mc@g^YIPGZ#Mg#E9ZL;xMvE%%dmFF{%yOMV)z}p8B-DnU8dsl_fHNg? zu|B0O!la@$QZuNPKCpvV<0$BUo85NyS-mIi<#sA7jnXt8ojmPsVLVB0ybd46#S0-1 z!y$=u15bT~OZ1ET5ee3)e!^6EmD-uvnokP$>XmbXK5x6oj@+=#31r>F@Yr1u%YL9( z)M>ou1EZMryz0I_t?~VCoJr6zK@t|UFXGP>H}NBakvbqBrIO`kcuPcc-ehZeHj z&2~+jgGcdk2sh6&@jH&`@^pAI#(cYRsAZvV0OutTEDr&ZUvF101=;Ij2OD^o?!YHB z#+c))rYPOrjoRrS8aap;sw=j*-kGRr0pqFL8uOl}IicLrzAA?m=8azq79Cm`L{M3J z!=D*gBPQ|0t2lfsj~~-*fL1L@mRL{vDLxj?8x;rNFeq6C$^5%dmONv`EyUS`(=!_9 z5_t&R57ynTIxk2A*yEzfO_BJi3b^z)iWbGOTEdR@SOo(^_}eJSnmcV_90#{fv7sMG z4JW`pbRNad!V-6#J1gaWD#v)sO)Q4i!a#koSwU#uo@TzMy~kqN1Nam&;0wmo!zuI}y6H2?15r8>K6VQW7$;w_)4}+xX5ybuKdlNs_ZG( zd$6RgKIcY6d39tOp-1Ei(cO|w^|z7hc(nGl&H9s&Dc0zPIowe(l`v#{95l=H6@C0A zk#hPEJ*MAinmLNyeTFAzx`<--dX`4tmG8A!Peg-o76#p(gcI)N*!L*kf25D}*XzFLINwEwyzY=9`51vjChtArvp zmalOcb@dqsVri6|jIUk*;ayFa^HBmx5)ykM+E99vH@9g%G}_4|Hc`(t7JXDc%~lp@ z9@N{*;cogu^BT*$i3^Pf)2_o$-U?@e!8xVg!je>DaS~MrlrfqayoOVj%WK`Lp!3|J zM~Z;?WQrxhHJPPIWT9B3q%@uo7c;FOl71HqMh~V#D}~d1aip6*>AL1hH#fV<hM zZL)<#+tG^vbx6#f5`tvP)^OHQx!J>Q9mN%d%SzsSN}qFq=USep9CpKOa*r>+x7Vix zm!wv=d7M*9XWVI9D`rdAXHS`K^oyKd@b#Sk6NXo?s?VSDWT+rC+JiT8gh|C|WbAhx zO2+DBYUD7So)urobT@vQ&V*akWvOo!!Fuj>=PJ6Z>`WMy<*wb0`jg=Tas9~cstea9 z6Gd|CvXdh(t45cc$L4I=z5ZN}2_hP^;JCf}_BVL^s)43VER5 z!|S$pm%NgJ^q(zsXwkQ$a&_154-Kt5ue_t$%O#Un;#1ez77QKVnH(5XJ!xtvEfGmW zdmW;c#FqryDU6EZZX9yG_;A1csJDysY2Y}99n;ZEuV?pN4ViA*i^`&Ly3E{9cyS*w zn_{*_!zWoKO#9jre&w>w8i(9SD>i^25Q{K z*UonTqvGu1XiH5bhMfgkjc5GBc8Nw|c+lDYtgqW)4 zDv6Wgq(kX}J64ZkRZYj1j2>e$iNN&k{4sKnuOKFeS$ZqMhVJ_*GZycKbSZ_j*OH+pfKdG?R(@;L%UM|lt-SGk+>MS`5eZaTY%d*Eu zvahv~hNWN6aygp~b9t^(=+62`DmLZ?%D{`>){k=@V+iPcIXruZ9h9@VMoYEjA{UAn z&G+x#xeH2Sw54v_XDh^col$HTS?v3~cAT2geqU&-vUa)K1ugxxy=JpR4BCsViusx< zskKT!I!}}jT21=Ru~GeYS8oyWi0yBy41%rgk%}d;MTf%||0&Nb zo0*5K{iiLCOYo$40e#8%vLG3^2fx#zA8$iT$)yo;F9ws5`%{0_+5sNo_6AD2WSuMD zE%v^K%J;JT6~kS7WjO&Yp$6kk1Y^h#>shY_8BQ`(8iu>xH|9!gJG0&$+;~Hl*I$Mw z8k^QBD~Yd+dx3^f#pyIGz$ppRU)2kfJNoiA>HTC%T6clR*ax@c>Y-l5P&fMF`$C_o z*qEYs_~D`7Ej_PxXc45L3zdoNV2pTYm+NemH&Z5@1;aJ+_?9#&LM0Cw(Go9>om z05Z7oLxTAJ2af0|m1tW04CBTDfqpk572dq@DQx6r`Qm}IxJ zK#e)o?~}t~xM{7gH@$YkD{W?hzA+Xz_mU6e{Pmk8%!CuR=VW_pA!8V`SkrO)D10D% z?fU{Kb}}hdz1@PumRUL#pu_zIiRu(vHhqWfI&1-G+1AJM_6tl1gC_f`zSp}RH!W@S zBk97+L4=8Sn#k_3)gNOxQYRX_v?;wmm_y_cN-2z&zw7xNvC(XBAsO8{uu6^7B` zw9_3zKj^!E-;dAlq(4DC@>o1PS6j&E#t(PI1peBUN-o5&)d}w?k<(_mL8G&kPUOAt z&6GTi^+6QHOouZj%>-?t-(v>P6BrxYcyEOErUOsnmNfe7aU$r{Ll**^V3dV$qW_qGyjkrtra=L>i`)2GACi%71yRc z!LCDtEfqhyiuKMC+%sPq^gN>FL`gYOb(1}z(}(>peC(FP4w{~S#KZ?vYiql_pWSiU zdTBH_JFH$Kb5g*1nX3@g_QGbjqKvL}v!HZ-JDkfe`F4LMvf{b5X0rrC%zy|_K)4YHY2!wIq0RqVZG2HlxaDJaFvvG4Zv9JedPw?+@ z32d*6#z6ox4Pl}{fZFZH*TDDbvfmDx3#gxif!+2`(173&3<#Wl z9`yJ5GJv55!p!dG?qTlp3&kET9w4rO2J26#U#UHa564f(_nSo7*Q)w&g8B!c2kbF0 z!0ZBq9vCR_K=?gK!TpEV0q4JyWq%gd|13*!JD32hjk&pvrKN`(&{N_7lq~JtfCdv+ z3wQfp3HBd~5I`03^$|ea01*b1*na*Q{!WPf^Wgm&>;L&PfPj7jh-T=Y4*{9x47eHn z{6qXb?fvs1{-vkmS9)Z)fU^w*oNXY8sSu#-=Hdnlg+Q?w1eCXb!uJFyfBx-A{y)e^ z&VMBzb>$Kk!PxEBTHEoAib=h_32j9#L- z!{&0uuRnJsf`XffIjj^-l0d7q_T$5|6xz8bZ+2+x-G;IVz@ zZEmIp_9)$%qVi|G=a$Ef&)aRaO`Lqa8WLL=u4bxcm%GW5_!mOJE9{!c9&O!D^%8q1 zrhHYy(~?yVJh?qCTbki-6_&-q3@e^b6I&njHwUXqkXzwdI^jG3o)kxps7 zpdoxM@4cnw&w~smFR8If%#9UTENr}E?7AN{hC|0vaG}^%wYD}a_g}VR-MoEndO*fw~8E-cwaiXPA?+J*z?5+IM2Cj12c2-kCP|Tzt{d(b88p=_)4_?zPA-mqeRT zt4_aGatgGF!WX{(@<<9C70&KbHzAc&J2`Q9+bR2VDp)&yWkdd?`RbX6%~?La#{9OW z%aW=5BU4Kw8hJJUhNh+!%CjbuBI{?88m}ArxT`7%(v-3>O(v5#%`p zNJB*Jp)B$rKHw_ZN)RhKb_`%-?rxZ0)IdFmYS$W3p2<@}D6_``m|Bt!145%|p zwm@-r4Nian!TsRw?iSqL-2%ZqNN@=RNRZ$fTm!)^Sa1#Q5Q4n#^i22k+}k~`GnqT@ z{p3$ld!KLbUA1b}TAAt?G`fWfBRXm|#2~=eD*Mpw>IR)#L;6pOFYY?ghAsbn;_0#*#@FB8N z(d+Ld(pzal;K4ElS}^K7?(`Gvr-L0Xft*m$>wq!nw1UQiYM2fXUBn@uQ(EB+E_%l= zl1`YR3lBSM!h?T5J=03k>GopApwUF8ah!^{#|o>w3kJ)89t#Z{1u^wWY0@z@GU&AI z7{q6kjO^D#G#p*Bk;O4#KAb37^fj(IS$8v>Ia zszrcUE#q;O6D`d`fOuPzplnI?XhJ$Q#*#`M5mEQ0ByT3PiMSb6S`178CSnVn2IB)E z)lOKDObHLxwVLXfWdgZHRtFZ=ZVXH}JB^v;YkLet3LdfP0fJ9M*?FUCP@~DO*^m^9 zU|yN+QA3MZkXNFF#E3feSwe{n(TY5=x_OU?Kmjc>00TqKhB3=S?&OV#KOSLtjHn;r=U1f}5Iq!2-c zfaiXy=l-HyK4u7XY#ibHg;-Xg<9=r-w5^F4+zLVA&I$Y1Hpl77N^{S<-0tE(k<)Q% zK)by=FPr+}6YWa-tX}zyXWz&iBDU4PLvk<4+^$V%wk?yi$~Ag=49<6box5qc+ViUX zZNl_?UAUlnd|Ug%Z3~$H>g(A|Ki7ib^t6=}I_3d2~X> zLpf?eJNsT8#wBQuq)JG9zDJjrOdhJ62fkIOaZmQ=K7Cj9zDvPxs$A^-1j6krAh+0a zC~tobA3^8_4f%yE;JK9M^;6-+=O6PK`9rRmgb>5?#oHiBxoSU86&kzbsY7cgScy=) zu7nLV-iOHRo=PgL4@aDWzj8F0Zi|&1kmQ#~K=+hVeAGQDo#r4lLuoP)Cw(!;TkNLv zJxCCvnV5gf5UL&~qa2r%Jo;rd8pX@8uZUuVP7JyM#KlLQdo*e=p&#)c<+ih22lyEu zs|9BS9K&&(y_%)g9*!>ng%@(Qc-6R;j;#B^nvhn!77Zq+kV>0$ML?sHH)Eff_((h_ zRyZGJGHxU$HCWvBRy!{UX|iQsTi=a{EJ2~FRSIg%`${1hd0Tg6O@ypBIO;?gZS`V6 z=5=*4rwXd{NrV>f@?J^TN-!&8*!3DuTx3aE6-hjFFiTJ`A`ES1@pA`CU!MS@60b@7 z4@-wh3EeNIJ!)!rc8r%hWR77~M7mDv_LN-R5Y_!26ZLb!WN%fu`tQ9jLMg^xz7MhK zukOp8Gvwu9J|^%B{i3_3A`jV++U0>gN(i@!&%t#i-mPovqh+`tcWTX6e4#!@dnB|9vKFuctO^$_q=+8kvF@Iu(yo@u62<9|fRW1yz|Y z(!mhwo_>CTzteTfE=BCH2-j~mZ|XLDr?g#SVHZpmVO7>1Ida@qW+T0V(rR-PK^aA; zcx{^Eki$=$^43o@1Yrgd=dOo_`O7*N3}3V_mqHuu?N0G%uHlyTz~*=`r|+jpV~im8 zBfAx%N0qer3_B$4s8>wq3ARpAZ!9OK&#y`gp+rO$D!C;cHnOqhZf0~9YLgCmG}b2> zR%|OLgSf9Tr6!=uOU74@XZE%dK*LNrk8<`Okt*XRR?#2Hic%$)UHBmzX%k7PiBGXs zU5JRgXs$YsHylSkB22)7{+QcNSrF(Frh=kmy9ctBc3F4D1~Dv-!M!*3+caeC#;>Ry zO}ksdFezDk83{AkQR|>RnpmM!XnPxH zv#dZZ&B&x{L>kr=S%t^QRt0?210 zio)Sh(#`g`PV%drXh(b#S%#EeYMSl69TiGj=heF;5}vBt8+S{|Lqt2b(AD+FJ`LqrJ6Xxmu2;%v4uW^KE z(V5Jv$s!&Mel3Gq#-S&TFUNT_X^SX57D3;nuFxS;{sOBtS5(e zF0WS2a%?zp0}Y0lbUKs@Y!LXAR8`XAtf?H=;5v>SCK;JAI6*H-ucJVElTx2;T{0L; z8MJzExHmw_=G`IE2xBPUP)7Dw4LUWZ`4@so69*zlOq8?(uO6c;lCl`xcAfc15USBg zl{~InbHU^Gde1@A8>HmmWwLaf9+{`W56g<)!>K5)flgvTrJx8Ms2I+5PmU#Md_4@2 zfr!TK($M0qchfHJ?ZqHG$c8yHph72|Pxpy%E23{N+9l+&(q(=8$u4BdGQzFI?h+Zl zXJOB1AES#`XYG0LTee~fflPOt8X7J~jr@pKjwdI$K^Ne`o&?==#v$ajn}Kb)9sQTMJnR(RAa6*v0<&^C|bMjeI8a zq^B}rSCkJ>YfaZzfeLX=-d2l^b*)Db)^%MU@eKIKG9$l3fF_0Mw+2`ldpO)LGrC<_ zQ*+XcoD4o9M(v+stvgAh;ub$FV|FX_nW%rd6(d7iz&O3mY*8;}3Z$lXQ|&86;!rfN zIf`R~VIBhh@p(^BzrgPL5$W+=?Di3^MAbQS^=+}%YP5)47zqu-bAOfzF*K*Klf|wQ zY&DjLBv8VU1D}V%+G!qr?t7m0aEy*L*d)m&0J9R{G=?Z74sS1v_(L&Vl z2-Jv`WX+=D@9Cp3TFM%E5@+;$UMR>Stv)Aw7q2NGR2*hmkz*wcvU@DQkMj!s%B!p! zUVd+_4nj;B1P4DJ6P`9oCPth62J;EXB`=R*90xBe_kR211LOxx82CTH9|5Y#KR~Dd zOZ@R)NJjkL*YDqxg#eiAzgxy{^`wQhrMt^N;K-kvVFV8Z0$dah5Wq17Dqs#Cu$VP4 zd&dn-PyE0f5<5_p{$s;`i}aWB-4dAXV&!&lbTD>zb96DcbTM~z1xV9B0r0ScF+Tt} zV+FJq0NjiN7<}dgFja85{v-TYK#+$CneB^!VYJ{~SGY1sg?z zx7x(c(bU${+ST02*u@xNK-rmr8R`Im8q9n8iERh-|9OC+?O&GwjI6N$v(P`hgg-~s z{_mFn9(-l}?aSf(Wz6`;zZ{O=18)BcKmnkB&_4qxI&w*0>M`5)w1ytTJ4`bvVE9qf z=+vtz_CDoXQb$uoBS!NG!y_|ItSKF?7@gUHGt5XhC`q>ABuC%~l)RcT1O+cTE>yv2j<%`3(_pR+ zenD%hB|V>E*qdQ~;@Tm0dTvR`3fJeMZIDt)e1*2jaztO~^vo*}zlcZ6UxRlbvvA7f zscxi4u4648m0+Lx!UK2ks z(hYb-C-}J{HNU)1OBF6;&KIG`*NAe$?p;t=EVDh|<%GrcVS^gqF|?>#=YKS5(>iQKK531lOpZw}Iw@dx=s zNe;Z34~}WdGMt@7yW&6IRG+pm??~`uy2&qBxSNx#UF-)fPxm^x5Jeg}Dp8#boI?}c zZTUIxlqwH?^D3*+wAwZLw05B=Qix|H80=hHZAOV@RWuPHp}1hsgqic6sEHA<9bf&b zI#clln)sa+{%7d5*rrQ&VxQ z=h37Xh3WC_7l>!2kdUW==-gsXgCf**65bozx)p_=%K2@_BCovI8p!F#0-H&SYABR(x<*4pm7^5$U3;N zOe$Y4hVp{xp!57$)7V++jGxRpGaDN}p2qEEQoF*h5H&MtNOmS&)#N?8& z#HJ?DTPW7DhK(pmEwln2^2%V90$u0VMwnf8&%H||#RV;6ZDa0ZoYxGd;RNzoF5H<` z55^bAn%DG_lWvwtbD}+9n4c|4a-FChYUEb58s{UtUs_0zQW6&NoJF75V<8%T!rHUm zMx<1IQ|CHrMeX4RO;1^P$1oAyU*LXNw>8<%7{vIV-4Q*5u zjpgwc=4l{>GhU5k@pn5PmesxqOZo>0wFW^~4z*>JB9FOo4iGB?!YxaxGAqVbJeS*x zw?aCyp$H$#{NI;HE)K#z5XixP!xf~GhZnQYBGt}#sb8CRD2>I|{7ONkf45T^{nlgu zy4Ojx@zB8GOUwwCQ0#{AdjdmVxie1fg;h}`NLLz}3~RmV=jp{32}&Lfi8pVFoC!zH zAInc`1bJmX_WvNL63)ST`+f%pu8B-y)rTD?b^&V$X+Au5Z>m?7w)#<9Bo(42yfd8m1)9O{8#os1hUfIhbSp zRj$(XiJ4k($U6dSR{7I8_oN|els=)G+E9+uO3^;%92wA*aOzshB6GlV+Ptp=Ev!%90aHw}uZ#RN*ybenIT39}H&%J++CFr>w6F>*_`boqH5f-%3pceWFZM9^{A6UAt1Q5k{{z6w>5Xk^!fJ{&726O3Lh)XH=tX~?s+ppkhg4d=bi_TteBJuc_jS3OF$ z$o3v#z0wK|+1Nt@|DCU@@fz%3@x<@hWLdVa_GZ&Zs*?KZB@2l691NX&#NBD0uxXHr#owdn(mUSz>9qw1`lj=?ztC zkeDAjMZDWu2TIt|?C==M97=}J($Y{Ec?kqt`<*(+VN~+q+jo!#Bc-Ri(OQ9DEK4nW z3)8+{gX}0`M1>S!M1pj9sg^A~+liIFB)Gszgd?UafV8a;Nf2H)*kohZQ?{4j+vH=` zs_a%!a}IP!RCcpq2zN2bQ%*eUU3#WtlbL9LL^pJ&`~*2GVMQJK*~;`SOK*;CBsN8w zHEnR8_))7*jyWFvjAltmU|VtTbL6_3>i^SC-%CDlzA!%PC3{BIm8Fq&^n zgk)g8EtmVz<@(coW+#(4#utqGj5Ed3#Xn9=Yv>)4KB}G3iho4ha>~%jwDzdf1q7E; z6h2V59^)>~_0CchBWCx;cDt%UY@iWqzb9W6?%n)ELW@kl#evoOL4qk4h0(rdVORvK zU%B`PFQ#dx0ob+vP{3Z;Rlu5R6LD^t__cli_qOaZX8l|x^K`OiCEVom^In!E*8!>P zXnW?t_K}L}j1*v2GG#ltC{!MM=wP z#W(e2N>F${&rH@*?;%n68}+NwTfv-?7OQGjMVuFwgZ^x76Ca!~$UQ?h8c5!*+u1YA z-hJ_xnf+WgkeVsLVJ#N;MrEVL@5re!-u{ZhFaqi1A*|~bI;;{s@#TkdxTd@@-p|7~ ziB~hxucgCeP^g|Zbf^6_1 z(TkIEYwlZa9NJ_n9^v7hRSx5aj){QRr!OAEFKIR384a_c`=qm~ew!(1JASH)p7oUN zrb(E1_Fyc$yjLryV*!8jqyG%jyMEAgFZ}y-AGL~Mo~$DVq$W&SalG$F!+uyQ&8Q(* zm^7kSynM0APA5ISCx2RqcbaoQq%7be~Idq4!AqBMc@9L_Dn&4jS zWPH^p{!WKSuDlm@*bwh{4}WDBl>Y~y0#L5~qh9>K1QmWqYWfl2fLU9~#0IRN#Lo|C zDcQ3FW?NwV;1@2`KeQ&<0iwY_mhoGu{vQMt*n#;qPCyuj9heLRPb6{x)FE~t6n3EW zWdnE_f0@?+(2f2e(*G^s^D{kl@Ejlqc$^VXis1r+12iH4krRMF1emvQ{RkloAoTo6 zuzwptV){9KOm;wT5(MZtf{j>M*Z`pzK!t|~&>>_69Hw}HS?0gKsec7i`k4_h4pzY4 z20RQ16xUn;;=ut>Lcsn~tnA?!kg;9Mv zf*L}U;>sfRKwVX)q&kO@EQ=WM5o}xy7ZuGIy1zM)1kombrrIQo?vaAt2{>{Owswn| z*!dprc0s@Lz=5Tbhn4+R8lPg$7W36XcP@J5XS(Ttj{R}ZQ0p(pTj5;MOIPUaUF~(7 zjG0R#cJ01CUS}%1{8-ki1)&i+Xqf!Cd^t5rNq!B{=LXbt!})KyZm^!0#itrmp3XYd ztG=z8zr8=F)SfeUr8m}itWI%dW4+eSFR?d+hex60VZwRP4Vfx1J2C$04)VE#9_rT0 zzHsa-fo3%6s+SWNyE?TNtv}+ZO{zIKQvZ^M-;8;}GEth& zj|#1zF-JFN_iCxP{d~mF-hhjO(d5gL0~rw^XO8^j`RF-CgO$^3{COdQLuNE|2)Ryz zE|V5A+fn1Ft#9|-r8Q=}318zzPVY@kB|YY~*yoE5_bFR*cGru8gtR=RY@BJr(Cj|e zdLU(AoJi!R)*XCG^-iAjs(o%?w`~)pM%dsVI!Nqa*1oECeRs@(NuFA1NMbHZZ`2$YUrS8ueSZwH!#u#mhIDs0ANC)jLsx0=4>#rjA))?n$VN^%Ekp(rd$b ze3Ei@N5)S| z8J8F=tg*l(a+Kbr{Bh$Z9aOGD+yLeGj`YXxsLHxt2fcq@xdTI2i{FmSjyl)an4 z8}#zk;w6>q3h9$GL~`zGPGRQj_0=$40l{~Wn`oJmL(M6KBCREaSCQMDBj1|tQaaSG z9x=N@#xPqwF!0`OdO@8$E0m7uYMDR1UgeK|QEzsf8K1tyY*T_jg7WyeJAcKVrqnzZ`5BNO^Np9lD|EvFBEETq{pWV*pk>c+%9Tw(pyS- zYf{EPjb>*62UTSzV2YHhFz^O(L2VjW-6GcP-6Qc%X)aV>m5+x_Z^KI*?8D>89zxUR zM{&i~>g3{SsTT|H2tQc)$Mzpd4!zCm)^Zap9p^K5dH1x*FWY8Sr_1ttvY6qj8C{}& zUs@lE9D=fXWC2oCn$8JSDkki#w)~ABK6}k8q_Uh%7cAtwFy3n+ebbK#;#J}AP)9GU zzT#@{3Fr7dt-!&kvwjDij~WT7w=XGs^3vZ=>Lr@IRSdaZ8%%V&;DVxkjVDK6jB!dK zT)5F(aB`FHExvb7#7nu-*S^FN9D?V1oy^HAHW|^**s$NW)^4gDUwnG;g42Tz|32=~ z*Q`fRG1iH6%U*3REX+p@h>Gc?gjer}slM)Tqj@&_Y&whr7XE%r1=Vtx`zr*wG7Q@i z2GiT`%Sfw#JfeaX( zYdAU{=Ev($R1hh1xL~Px_m_J)&1mEi1Du3sM-2TgSq=KgsY&zZ2Qe?d(Oac;*z0{V z$)w3}bQ7rC|N091Gime+KB@ECt6=OZ%&bQR*0_qrN&4?FigaanGChqigQ(sVf<&O5 zXs%0Dbj;zbt3&5-6oWh+3GG)}q~x0>`Gzyf%f`4guEsDAUk&UvY8LAnj zBU$-CLqL$RA8Jze8A8HFgV7MYVUw>vFphh0K3_oO(Iy7wZWfJ3HK=X7(Mx{6V|-Qe zGv%F4Zx>X=&`~ib@$kZ%%ux!0PNc(E4a|?nxY}QgFlV(Tp~0bpsJbc;6a1%>m@x;H(8OxRtTz1pWXo=6@yNf7rwM9|`zhlNAG3B)>?h{*!ttxIx3p0a&8~ zO(w8#A1lz!1KMR=oPdWW$B(3nf7q${AA$Z%`w=XD$Oa_aTtEkp9Y|uq>X~4NL@pp1 z2P;ec<)oJL_YIjpt7(2}O94tS2rx$l&u;)ShG3zaUvgW3vJKYe`0IiGzJ~lQ&|mtb zR$vWGE^|{W$DioYfkmG=*}z_t09_vt!UT^&fVuF%2*)q<{l6aQA6AL~xj@+fw^=Su zph)~RP=G+r4b)pd9_SwyiGL6DZvyoku7KDB&;+HY|B0Oi8x(-&VE!I2*w~c?JRkGJ z8~cZK;(tC~cA&831PBDb1P+Z%n6+)V|UC>bJ$_O$-Zj)BS8s638>ionVeR}#(R^R_h zcHn5Hp`y(zb7T6Irt^Cne|O*Kp_;KG?u8YyxZ5)B8W!Gq<14GXsXHD=Hz1Hqj^s1n zo%sOn^iJ8)N)U8ib${JMZDwa3AF0)cSnEdXzPx6h)^QCp&)dnXa$E1?PJ;9&dtV%c z_;qh``Fy@FSS@s(IOxa4^@RhCs>U%*~a9#eah{0xeca*lmz|xA8{45}zV98xrt{e( zMNj0UjrBcIS{MDRVg%u|@+inYHk zWmUqEmjABcsfeFb!Q}VkO}^~9xQ5?nL#4&4z1EHzj+@TSJUIk8z7{TGs&@O2VYNy1y@cN*!pm9|2g<(oH0NPV6xFkB^Z=FB^} zmB$$zcg5qolqKRGNs!=s54Q~Z4Bf8Te^OPmPQZ*`^{M@|F8X!zL$3FLEJICq?Z_I{ zU`lD+u=MS;%v4DxVX%E?Xi8ozD7XA*lVyxkNM2DkihpEsTIMSE4Ra7j zqED|z*x-}|p~gKZ+KqzbgI7X`dMc}5ze~;HwzPB-Ka*B;i)^6hccToT5f%DFme5?fykS+Vi*sOIqAm# zikqi02xRfp**ExjzfTAlnuKrM^lT#wo_RXG-@791>KHm~ZF;ln zAMTjF7({1rqc08*YRrC3=7$y&GiDN2S3*FHgQ=g4{GD~vilr1){J}_P*|V>X|FeMh zV{v&ZmYV9TmJ*W+k0jq7EI3V;5nZXA!*Q#AUdlyq5mrLS{e ztP1I=9z{AqL~{>nCa31K8@_1SGJpv~UObye&6BUL7j2*6YM=veHZx=-#Nerf|xf_zZg|hA=^%N_E|$r|_3) zik!9CIQjc~`gcCx4vh;^NmEAj^NxsvAdqVEZd}aq{qXs_$&hZkpwE!UUVl>PQ3olb zX|yN}JvyNpKNEKnxr1z|{i0h4y6hur&%g^2nW#d~VRmw2Mq^LRT0e&lo3Hv2pT|8h zR0g5fdXkmIXIE@hulqiI595wV)b+Y?FGjEZt;PqmX z;j=^&)oc!&rSC~ae098d+EdjLm6#0Wn0)QVEb9Jk_JNOYkrJjgvb3;@0|I^RK|OBi z;?Fx#*(h%0FI7@`h7d+ZPrH^Sd;C3_7g$2Ayka4?yI?N9R1`6sD)=#*n9Cs4P$nzy z_RF9*mtQxnh&8!sZaP8B^cVIbDK!RY4>;}kbgGb+-O*^j*EX&@nPu1AaBQ2HTo6Hn|3c?!I?@46y;ZK*EX!$W*_5-U%xvfe3FKUC zKif2q_MyzT3H6Opf5_soR5OpMbCL8+gt>UX>(9)o=P$#G{C%=4%~F>-Quv|IB|y?K zBoOb#V-}Fh=>`WI6?@+jDoHE6isFDg2X$MF3bcMt-NY-#zS}K$aj-2zu-I0tG3Csh zNiycZcTAe6=(K(Zdv46^%h&y)7)q?pOYehH^JWfp8WLlI@yQ@}-gba48OB;v6GQZ* z>}@4alkxM);0332*scVmVTQePN_47kJ2Ezy1jhG})it{z_gB~Nr-g*=}7 zZ7K3SY#9ax%7fR23eqXUYop;5*>$#8k4o1z>R|fgp1Wt2`-o@tm-Q7jr3*7Rn;8xx zcVdu1S#Du^$`-kvO0|Y!?7JiM-HoHFnK=+Y6@Nz-_mPJrBf280a@jSsALnUMc>y;< zS=XuWafF0wjV5^~K_>Ras3zK~8h)5R>`5Pw21~uVbClZcvyU};#XCoipBSLfuPEZK z8Zqf8w!Enr+V?`T$oSp93w#Yrd>Orz`ynvdZ|o!URHVEay}Qo2pAzqNx*LhA40dB4 zj)Qb`=tGNVz?p0=dpc4IJH>?we-!2mzc20msjL)Q(J54U(*>egY0O={FoYhn96>0BsIWjR&V{e zfk3}K#sTXUj^`eyVOz_V?@H_9gQt@WyyfULndoC9l#aL03Jb~?YmhU|U@G{ZTq2~6 z7+##UVem$EP%6GT=<%VeE*_jRxfLih*q_5(TH8>wNa&+F_gN)my`_VB?-ckTK{&QF z>p_IJ)Ssz{dl76)$fUeP#Fap2#i~1WDzxneL0g@Vx50WB+ydv<&>>RE+Y4Q=igGZe7yL{TZyT1#rE zCf&GO3wKYg&^Pc~X%2KqubvL$ySNT$%axzub+sn);EY+-*Z_yqyKoGDoaTcbyVvo*876du-O-JetlA@>bOn^M`H&ZZJY*%O*? zO*ISy?cbPQhdyUbpQFT2-PhtWizug&*V&>sH z?JCvJ&N{DLy+p_RWIJCt7^C0XN+_=$w%U~*dm^9PaUOcSZbPpd6P{AX)LGUfpcCh? zDEK5dDy*v3r8Oy_TQLqY8{r~$yIjOp+l(sCk4UOTy7*3B#;<;TI#=^T9Id!@BO$3c z&n6i@d4|GrR$FKENe@4ai%-mvB3Z_Wop13AA-I{rZq?|53=D0YFV|E(7@_!3p(6?H zTJBz2U&E7{i3YSV)6Gc|Jeq=uuH|Iy+-t>O>Ft#kU}?#%EG)^t@{eIufqW877k@u> zx>9=t8e;x%a}t*|zzGjePX5#o5f{HvW{-PnbjCtkHNkQQ3G=i?CoB#2${}>~RZ0q^ zi+;E?_uT0V3L#hHvds-g7W2#FuzQC1!I|p%=3^;l-hp-gTUaFp+ZuR9`yrMru&EgmMg6)`(1mqe6;wXBRin@AgF-*ZR-E#GZen?x`mLYL z(eRGwQOqaaKLCcgevk3}*MMPmz)|g2O__fmCjK4$0+4t9M_2nFrN6t$uWp|J#PAb! zH9Npt-~jtv{DS?0U3-9k?-%z5F!=ieiYP!){s(dX5AYZC6Opn42oG4#_&4PifHne9 zM1EAc1!(>ML7acX(~KSL%)MCwAzFKLV~3wpkY@#8Gfn_V1w%f+Xvc%eEnrpyFi`+t zEq@)b00vh68}YIMYzuAx@%SxXK;r^{+Ww7r{}lfFr%(1L&0hhfYv4-&1X=)1B(T&F zAW;EUVg^I*e;HH*JgWX+XMa6bKSOC@Wnl$`Hvq+DK!G1DBm)Tc{L)up0VL&rgwX<6 z5d1-ezkI8|`sNw`Vq5s@UmhG*fF+Kdx4DIdwW+nagPXmzogEpVEqrT z=dXf^tUN#;4Y<@@-CWF#?UB6ZUunxZZg8QrZEIau2SS_a4W<~j7S0_YTSYreTvi^4 zn9`ea2295f6}7h&Nc&o3r87i9REf*rJU#g?9ecD-y!`F(lc7KC?G?#tK@k3R0ygq; zlJ)+^#k7#tn5?-O4Ke4zSW677;hplwk?)tgTG!8PmnwIM6Am3o%iC_aB2?kU({mBG zSI2`iQ%*8^t|quW^5wk((sOw!6Y0>&O8^mMbI_zE;C=!h!{f_ zS&ocq_m;uYpVy9;-mWbg08m?OPBnGwwWd?=8TT)B}K{}Sz2#&a^0(n-D-&BI65GkgS($fRypNU zolrHOmiY!pyKrXIpOk})X&0Y0E5U4iZK|qf_fSYml1KGv6wzTDjbFr@Dk;hDr5kmJ zPuVIf)1qt$Qz`9IXJ!Dsj?8$LU1~iBPhUn^svRN5ob=*Lzu}@ZVo7l!+4zW5cKnvk zq($4p#Y{0Drw`9K&En93YV6c}!=xXh8fOL}O74uKHxX}eyGWBkIr*s^XR_<_(i=## zuCQ+U#^GgjblVLXp;OWHk$$;EZnF)>VdQ?fwulzW#qF1#glt~3#)O^kxS?C9T-TW) zlzeo^6*aQpm&V&Dj$5_MH3gDe)}{0b>NsB!7)l>6z0oXk7Ys`x85m-vlyfd0U^eOJs&LHhi4n z;@icy==W92?0az>7Gkk6N0RMs>_Tjw$IC({Z!fEGx24-i+54`RR{U${B_G{~g-Ic-=M<2^ zu+BKKXlf63IT{iIF13pj!-T%NX2!s3%&<9uvFRNjV*6A;1%MzZCvP5{^n*ub%`hqKMnGQmkQFnf?Ts zO9gu)^kXpvV|<|C0xiS?YDm=pE53}7Gwv{HfBjh=M#1A_WBL0s7d_g?A5qUqiVlhn zFb;9iInEgk2sNI~IH!aeNy-~bfkwkb4Ain-yA9ZC4)hyVW;IwyFu#0cYoqm^t}SO6 z|E)KF1UqwFq8P<Z59^C8A0?5@@g}P1IGFJ)^TAjuQH7nRNLp9R;bi*f5S%MP|6@; z-)Ed#Pta*sTy3!~XEE$jEI>*+&(IQFgi%>s|CSjiyPJU za?&o~Q)6;xfjKHqGCO{G7rp^?kNl0^eu$of0m{oHRmKo}v6eWrGI<%5Jll6neL3CF zpN3S`S>Du%I?PTW5S_fq9;Dks4*>xwI6|Jw%zUYKcR|x-P5ZV zCg&(^Cwb3?X!=T%k)v7N2HsQXi+>{qE=58m5x&K0vrsU(;_~=2Q*2nsWN(Z#*TQi3 zW!0~9OLyG@T#3xYyFo@qs2DLcOfZ-zGZ7)<``i)2Y*+^r2nMg-G_`fYTMm)a7Wq}3 zcfC>S>FV}`%OfyM=L;jrnYiT^ydwVA6JIkX^Z?`3%cra43>}OAru0R$*#pAcY1`N9 z^hw^UqOW?VHCD_MUS3MUSz`C)yH*|Z zh3eioIvjcXk@~~A2kR4tTxJ@=t{R5H7LHBA7bRsNrv%~`kZ;t=?fVFvFq|aeNIUd& zoSadf=+xOhZb+>+5$#k=52I00$Qm-hU?Wc>beLzu@1o{FwOtUsP3Dv*ZJeUr?_3GZ zmXag%HSWI%gEgk;pfY;fiF2ud--x?;kiCTR5`$y7>Xz#m=B#JpO4svBSI9mfmCB1q zeDHjM>>eGvGbM;j&&dAEG{&X(jp3nfcF@lVE2THsC)3 z0icr7hv%5my>hAB;)@dsV^%R%P)l+7hnR$3djY3sXSmMRxYp# zCs2<5^%ebj;rVO&{@+RF+1P;O7Z}9oS7|7v>r;{pF+ZTh$R<3Abj-()(uz|SN2 z{3oLYE8zl1^w)5~a~NQKjvtTr&#P50Yp}C0r@a;6?amAs^GG?WI|B94zcP9M1DYaG zt#JZoI===AK8XNp>E8(S&&$yNNTA>Y2G$h*El}Wq0SebYzM;P_FMosn{$afIGrS+L zJ}(%jPRu>ltsSZ$mQu;b$V5qm1&RQ|!n z{&KPY$72ShhqwWoyWe6453~Uv?vKa(=dYKAwTro@ql>L8aA~odx|zA3<@@4uWx0GQREZS;RW=)XNHbR#0ob`0s>OXkq8*=?J(k*aJFN#x9oT;Gf9d*z#wRPj(=E;|3dm z{{5%m0N;&Z!t9TJR6Kw9Q2rNI0QL)D0p_WHTLJi71I>>gU%~%=KL3KrqNk@P6SynE zXaV#kT$oH9n9UuS0nIT>W-DWBMms=;;OYNMD)>vc?4Kr9xB=ELP{9DjB?vqy4sh$a zfj$|~y5t5fs=s_U0lM@5xJ>^g7Z^-9|2=&F{_ncxqZMPurCr4&<5bs*Hs~YWRSo-5 zo|e^fb=lD3qF8KHdB=~3H)b567guZxoLlO`pC&*cxIr%(K3{y~Soh4kl&pgP=I-)v zx0fETkbc>fo%u}5a{-sXaO~^*TaC3QYvPJT78kWeo%e#g2c`>=Hu_O-Xe(wIBC`BO z2UVx|#&@q~SG`>i-p1KH+h2a?5HFzLZO*tyQh)ZL!$QXq)-RDKeo zuIl5Nh3^zd#h=;+Iogyy_uu}$0PT2KS4rv`&@>_%4u_M!Ufm;7;#gTV>^%T_f|_mz zjZy+*XT-QrGF`%&%Mq9}r)@W9>ePOq7f$KF zSPV7T>p#m{7>6-}ba>*c5e3h~v8?~y>F_hZQA*7cLEJOUDmPrI$|t1hrzE5n@^De3 zu*6y24J2(EB^c(>qe*A5Qku54$SOuhGK8bCCbpEU-Vxlg&Rt<|taNQ1SU-dt=tp$F z$GF;i9=fOX#PO=}h2{d`>6-95x$|m^O7z$5BTR*Jn0reG`R8yM8O(-lI=VB%z39VJ zL#8J@!5o1&9erIxP=$fn(=WBw7Q(i#h&0p;%S*&4pNXW*9yo^=UuEAwlF8m95QUox zzQIX_4wHx$Fb&G|+JNaIcHB9hxT6>rd++U~=$bW3D6*qo(cT@t4SU~!!@d9f73~Kv zJPtBmbp92kXP=-}ry`!=?~32)!#fAD&`_2xDo71sIP;v8>KE*t0qzvf;vw}&0( z+G=fmnJFK7xSxMiL7OX=mF<&X4@>MS$mK>cb-t`S#dLskD2;=Z=RP= zZ2gY8oT^ysTLZpX++7~FdEfU9{Ebs&BCaWKY+^_W^wW5DIvCqFB2t|ir#eogOlvt1p+E-kHFgjv ziq*oipQx8?@+pD~@c85KK^wYI;X&A>>fJ<$DjzA(p7oQy4NMB*<{8EO28S@T_1Yfu zj2`O|c{)~;iT?^?5P7)WK6m4X&&UZ3*=fwf5qDD*;5hBzCFlP?rlN$B2>YHv3FdK^`D%c zauj1D#47GE-=le)s*Z4n$RVA2%p`YFcEId$drXYE*uJ4l8D`K z?ugY-k;bYi%tP1KV}xb*wXSs>1`c}Ox?-waQHNu%Wh4=-xQ0`Ev>;g$4(rAM^$Yi1 z1B@MJ=to>6Zu9d3XyV(%Lw+27E6n)EFfm)$a<|BaqG%ZW-yb&zmXOD}SG1B-KNsPi z9a-sOU>F>uEy}DR>sW$bgRb&;&6UKxE*1!T{_ML?d$$mg#nvSp#s{RjGOpt1I@`Wx z6ML}}g~*j*!e@`VH92KIJHq#ip+XB)sFfwn?hVcGi=i`%m*0rVkP$H|u+Q7Fz$QS5 zKw9NiL6KZn6(2A)%0cu9*3RAH>6dq9A#0sd)^5Vs>$JxYemm)3bc-)%^6=%JAxA*5 z+r`|$=MFSYDwa$ceZEyK^}@;=q<69 z#vRkuqW-i>R#|o$1DHbR7^hM^6TW_%cO`C&!@2Xnokmf$oeYC|K3mwlz=BYF|b zT%|DO#g+prA>OeoNSBJQKHITS+4Z%+>7r7j*a0G!bDZa5ON^K*uhCtKTJt(?x&=1|dULZM9QxDf{L`8Qa2sdy9Ai?y+mS0@61X)WS%_L{e>GwoZ&Ik=H{-=(p~aGHt1_$CYgul33(a!Y#A7?7oxl3sJa?+HdhT5{+|K(<%BiEG z^G#>Fk-cHGea)AVqR6g{cKXZ~OFgJ;<44Qav8k8D^(@FI0yQf0# zT%M_BRAiml22XcKckLENQyqHR(R8BIus187a>nVn*D9Tx-a8s3rJ7L0x9OHQE@dxX z9rR3^NpaL_xMXf#Q`UXW{lx^8m7A!?v5Hid02$sj}YVpp8zhlXN@kAJ;Cxk{6Ddf1}pl80!n#**o$MJ@BOPA%^)m-4o+ zXD0&knWp!vyg5>>eUrC4`*Fs-2epG@WW`|4=v-kqF3rQz4k>QTs@;T$?i&+&>&v>VhnK>hSW+~lY zFqdUkgvzS==de9xyw&=`L=fg4lO%yoth%2svsIe1tQS@|Zd9L_ZR_28K1yDNHSsVi zs%0*ol(RK5rt&TIwUXSnV3MUa_m}a-U`uC>ia~+C)LDReDY_Adu7=d$m_4oddLYTT=T=)h@_d>sJ?kVVi~|W z5#ty;NquC$c{*+2X&wnSnI5NS$3iPcS0#&YW@m2+Jdn%LpkSOQ@ls-sw&caPioJrj zw4xMug&rln@?h71!s_9Zn1=iZA=g+-D^&L%j^h0r5?>0Bb_ZbCLZ0%n2E~j$?^(bl_5VV+~%J!&eQ4isEtQKH_CtJ zTwMB><2Q73;F?Fvm)J2f4zi=8hMn#ulcf?%b3wb)+v|H9dOba|xyOs#8Lq{49g$qt zSIO)R5{oXab9$cd3TwChy5v_6Z*dBjf4kF%)%0q^;?vG{sEC67wt#Ouyrp^K}85!r}g1?$gTB7;G=n(A3V} z4rjZeISJqM{Yb#87lNh)0>(O|bcn>D1Ta7q=64S-5how#A!^7y(8`!hvk$PsBn%$pRb(!V&*!?0zEtMHK`TJbr6u z@mLDL1rUPj3_u|W4cZtW-VwNGp!y2V1OaUf9 zRhtUMMl9Lv%x-aKuO`keRWRFg*mKWQZ7zSWharBm6N+hvIcEpX81|jq?OQT3>Mt#z zdVI>sNWwYup2FMMqB|2MRSp(++~V)LYG>U&_XL@V-a=PO@43^`Z;!txL!e|=Yy1~> z1_=Rorj)T$nAH11@?@1E?e357cQL|y&>UAaCHq*F6UC1nNmx84` z35Isk(6ac4CVX?yPUiZIW6s%q%ZLLOy`Pf#3ddIxeh=LBbe{ln-GrVqYO|@SEaP+D zw_+%wR#>_?;K!SiujyHbMRxAmGpagMeAuJ)@q_?J zamdM{ljUJmO|Lx@)!WdMz8rj53T7@5hyJ0~W1|5rXXg(cs;r=V89_nEAbRMz+FZ){ zdy88VqtnJ4^;TLBau41aA89z{L>?l*@YeG{rT@OW%C{+MzE#auNpBaQF_dj8SR71g z$Sb91%b>WG$84`MrP>+ML3H}G{++{YFJ})Yw#x|Iey^sRw{J_|k++YN$=^IV<|v;X z?%Q)%Zh5<3SW0DxGJ2}zv`VC;Mx0v|?ZVzOwtLcr7114Oa4#XI%5n*j?fsk%)0YQ#*iNnp&^&h3Kiw=I zdiIIj0YS$6i2`w5>9a#sc1{$xCg)DP>2ObpClj>SYgMOCG>A#b|a&abwb&N(-*C>#z#PKS>xH~5fs zs?#vaX#$~Ut(EnwR{nuPjn!uov*tp4+Sf* zFz;?pYMR+*OsCfK#k{&zLGe{d9sCYgM|iC)Yg1U`J~gRjtg#O)n*KozP(57 z+lcJozB;KQ~Q*}*_bVgQu-fuA}lh!bZB5mh$X1VPT z`ri|OJNn8}c8YX135nL9p7lTM-8a>tzkhb|WP;nn?MYR-%v!HfCgxwDN_{Lds1RSw zFMD5;w>t8qtl~-|F?GsToBWg-pOYpx@-zHn{Z3x)MpD_=D)7}_*L*NR^C~a%>&XW( z0iH2dBMa{5!xqo_T{`yZK=D=khk-j8QXehRr7e5RQ)VQjznAj#-?=rcG`}KuNpSY_ z@hT)Xs{FjZ!cHEgT@wAcdSBD@yF%fj)xnu(hni%4<6vVgYy!B%tQw=4o-J5!$Nh~CXXnDlLRY3R;kOjMhpSTZ0wkscjCe&95j zQFig(kn|a$))=1d*!hkNTV~>P?(~;Ox+TVb3MQSrk_Ts#uw$AmsdCQtRNummZcURC z)A2nC4G%}hkX`Hn#$2`zj41H>I({1a*j9mG%r6m!U|42hf{z6QME zn|c?M+I#HV>52!dd+rFn;d=Ed{@(bPvtPECL$L&ug$UkFuNv6;5S=k{;Jv4Va<${F}n*Q=|VCbsFsg#PvOD{ zi!{lzQOjyq^$sdAT%g#oBt-Yg2Ftd{P8B~8vA@412U{Fj;!PRu$-YnemF$+_*F;-< z)s2H(SQal573;_gE0b&kBM;^#fySdKZ5(lY=^3@I(Z#VH7}S9I}TKxT#hVfq+oh4afN0pylo|A zu_B(WUg6MngFJ`x@AMF@l+8>ghon~B2T5z09bi(mms|I8`3CY6IYqg)e4y#0th?+Q zlbLWw&Mvzu+cYNR$*u(bXPGa&RZ23#tWTUT_UJXMq)<|PEZ=wbSnPYT=MvI=uh_kC zbIfrfd-m^h+`fM}(<5UmscfP)4XG64g6r!O%iY{J;|fX0v(=?zWj@Gb75C=^P#hK? z6n8wpFID7oBd|!YtY2^(oBQtQc`y@t8CO^IoI*1Gl_ZOou7%HwBeUE>7v++~3fsI3 zy&lY|`uH56Oz0JO`^}HisL?cDN5QUOXX#( zI`U1u42uYTA|YUMMrUq3qVsrGpAbX#f;ZD~ce$sG&fk`a0$cgUO47Djd|DQ zd?QEbR>SkG%JH-p+l6T=NHvL)Mu;_qNy7vvG6s|_wJgaDQ$h=LbhR#oyV8_iOwhTW zWBMs*((BtfMTMed538AHSQ5;3DtoVsSugT;yBtR!bjj{u5_~X{e?867|L#YI#?^gN z2$M6JwgyyoPeX|%&khAOs(C7Sn_9s82$Upr0k~?*T`QFqroDz2X6=p={IV^$e!=wKP1Me zV((XdMp7;!=)*$Y!{u`#bIjY*MovExJz;(K>orT6Uz!C2XW{v*LfL;7QbsvSF|@9q!q z4L)l=DnLR1sQe0(le6v#|2CWqd==5UWGPTUaii3)tJQaZ^aWg;zWL`6L51ZTO%bT8 zgPh%uxwpE?--%q7YC!eaGQ!j7#J1aL-c@L+j1QVEF5mU>>t54y7FRiHNxZEeGKes} z+cidUCp$Gj=KbONfas6;7sUKH42{Jel38!1>3pBU!^a_*c(#h%i+|?bo6xb~H02|u z@7ztq&ylx_&lA7yj8n^|weFh|5v0ghG0u-@#Jw+28PhMn%~<=@aN$MBuwBpe1FCZ6 z?P5zAJHt*9Jqx95X%$g=FUEAkqQIG6Je?*{OQollWU2b#W2YO(qh=G8UuXo4IuGsN zA7iHG(jXVH0LR*y+kdEjdhZ)3`ok%Zx~|W!a8?`&dUo4v%T`Vi(S+hp$M2j^ej{<< zn0#YyFg>{k|6W!@Y2n?&mkgJScLvZNm}(of@!=Xx&$NE^2rjssZ<(4S+Gx3Vs(|_e zS7B2uMR(`juGl!Wk3{z9^cGEaOQckSyX<*qnjAlQCr)(#{_VEM&lJBOi!UJgxE%cC z%n{z<_OT|vh<%*xW>Wf{dZHRCDd%4Y=)59#*LhNKz+26N&ac3Aq?jPR0 z_;hfNa?Y}B=!(u_$&DTGGZQ=~K1fC`U4A>lM;EH^$sG}EyX%SQjBkS~TIb2b#wiUx zWH)&=hj=h?z1BPNT^d(8yeoHa+g3vsQNca1|IoGyf3eQD))Yi1zj4XvgvrgxrES0O zluA3OVk-7tbnhxZ(WSey2!~eE%w_S`XWWx2dzNnt=h~5WX_|eaM`yO76JA6{ z!$(&>c_~PHchzLmHXN+m%~ol$0xk5evMiXtK7ab z@atUQT+myVr~wkIF_)(Y9M!j19@uXAm7FavxGOI=AowgQr6Y)J;c8vJd9`=t zsiBnvC+|A#VBj5bui;L+m!B-hl-H+2e972O(b#!hA94GBIb7&Uucq5+=X$C)L4~}J zbK5CsNk-4?@x;bm)J0h{)us3EI6(ZWf3}37U`IWFpc5tRLmHYsfFn_LVxGHnKKNn< zr>??c*K+JCQG#~j?*E-fMFMixA0gYO=24+(6vAs`1wdoXX=o;eFyI%hwb`6T34;Fa zHF1W2nmI)RxynC)|BQ*(qIh#tNZs{k|HN-a#($lYgaTmz9Ek$Lt{}Kw8^#DFNzf4W z_xE|;NI*aQ_rU@cVxXD;&4BoU7IgSSEOo-s{$+;mf2dIi{nobrb;1Gky$b+|SCC); zbU={mg#)Be@(E&Yq6kR!B0+likA3~SgaaH2#(DgjaPZfZ3+cH}`(!eA>fqg7n!E?* z5>*oeADF&o;J$BBG`{bnPgF?}vly$`%U3Il&DH^7d~RpSSi-Al>s`Dg@`TN+hZLRk z=XeLa@KGEsh^gBtzT9> z7pXpl)i&sU%TUe~Y_I>YYjr>|?0vRxYNPnUf`*p(>$)Or*VApv!{7C%(tO;e7;-`4 z=?8zlx2$gitB3{SDCYIZ8WH@A^xi1Fo)jSmsks=J@M?0Nm= z5ms{pk%F3YLMQVh$?3NoZIQWt$Gq2tXGHPkgbkPPBqyzP_?P9SyoY-m599?mu{GQC zseRSIB4j_rG&!wHspd_`$+RWGrN!}+iM4IC`4ub@r&Ii+s2TcQB z$WSnR&Un84z(LE7Me4mLDZD&f-^eD7%UfmTW6$KK+!Us|Xy)qJA+|M-?nDjty5*pJ zzAD#q0h0Dlrh2iYd&%A~)fn3=xG_`d$eGnRMgHQR(x>wS_!*O&7* zCs;y@B*{t)LnZV>{qwGNJxo=6p;lfePVscfYp`ycBN=;fH_YbzXa4?f3EH$)ij&Xs zY(H0WH)|})4=P>kqruSLY-eO)5`IML_1OvOG}ZRHUBk@F{J8sh7xAg7Aq~q52UW{f z{Bi{Z1|{AbGEz2qhG{-t+hE;*JHaRB}-qOCsgh=YMT&8FVSr3_sUh;UTpli%RU;x zeD{9qC$S$w&aPlZ#ab8Jgw%9cSm>qi*>fSg(r!DT4xI7z^y6&ndA!Ac^2!6Z)VJKP zYGnlD&T+<{?1(pMu}~{CR`&cPwqr!@8Xv=npg7FF9<3ZQ#k&0AN9E@k&ym_WKM6Z= z8k^*j;qq7`3=RM6LLp;c`?KvUMjKQ$=N}6=tUx{vz{aF)Cr9#Nkb?7<~#xeP*0!Kv(>ZtxELm{)g}h9*TTrcqgmo>w%)>b9?w@ zKMP)E*sAJUKKnZ6bqSb~A&g3Taw|IQ!g4UC8}q zSCXguwkpzB%D&!8K1*VFmBBeqn0mhptL+_9_4(a`Q%7&(GRI5Cd7cor?K#~ZDJbcb zOsYXM)=j~2pP0&@N59|YSX89$kW=t<5l=~&s?H-lXO)DYT%|qt$-JDkN$(%j(e-)4 zd&9|s#G3}otf;%42NhYNF{vQrg1lX$bXl3!eRR)mi7$ENF=m3+$sg(+KE)#UPHDMP zBc|i>Gi$lUL(Qf%SOkw)o(nvjaOFWk)vHdSxO=t-!xpzBdc+o$Tm5zv{Du0nV9cb^Z-mdyMt0HX-q%w zZ)xrv-Y4FiPJh2LXYnp_Vetd0S1UtZlt;U+duNrz3Mps$*zukcO}#BcJWa8*5q1+= zQ+uD5qjjHYggJkML3KqkHB|?U*(>$t9(2pKPYNI z8*9Ud@$RjXv@`eIqrN+{N9Rx~nA4k>LbU?}bzKQs~+2&za>4Wgkgx?<0EM$?MUnV>0S6 zK=EQb5g@A$ByPVkfAHYh9YWD8-i@+cW=lQXxSVh2jNr2<1CrbIWZhw>JYA;bzLnP8 z_j2u)J)~Q1-Nsbe7}qVCCZwDlET$xPX}&sCW)Ia9i-ALF2A2cbz69(|%x&I(2JJ6* zL^y(yQc&e{ibdR2+fy`K=!hqIXFAE^^@NDcno3DuH*c?@GaqAeE)xT`?*m3}Vd0kX#X1bm)t$BltNfxAeGP8Q`8?{1%J%O=^L zDPMkTO+n``-SMH1GlGHpIp`Rl)prw zO{^iLd=EQ`74i(W(lv4QKlIqE;?|h1qMDZcg9r|)2OU=3{5)bE)|t7iBrWL%RYw>(qF-u74h&@aQh|CYe@2k&Rl-NS=$ zhlabKPoInu_k$alzrR)gv8`h+$)jhg;Q5QYTWnfxhWh_t>4M zPob%3sVU4U>!Dhz;U1aCuf!z|DD^Io@fJ{}9Y5B)Ok=dYlk6`d8Q8Ct(@l+JfZ{v{ z3=A^=9=rTGlKEA#ZUY(wK=*D<1MYvw4gik9FAna{DCd8;;=&to#nGTp4W!i|7hDiT zWRM=1FamT4f{ryn?D+kvDqxiSrYkN8dinl`D-PBEo`3;zINrES+TBE3kGY@gGSZb1 z{@@^5UlR+pgA}+wBe|>fwd;|Or4j?mAbJNGGj(FZ1x$64M|BcA+a8anUY^dYGSA?H zd>NWG$-}zU^oR69Pl@sbr5Kv38p64I_1Tx1kF(FdI~2*cVpa6j-Ss_Abg-$kTv$D`r9`M769d6Yv_Zl1V#e8pBZJ@D^w}jbrK_5g)8ZVjNI%Oj zRoUcA_ne>Ntr5AQZF^nW&GNO;r;P0E=w=vP`@Af*-H}4}<*~0b0d8B$za6|_THcUy zSGnWFxSEKlrC+1I;QeDvti0ds-!-JM1-c$}$(vKUU`?F9RMe#?UHqWb>^0V_z+=yn zqtf0p@TjMD`#+*J@+KlIT9~_D8ciD`jZvC~T@|$Vv_DoDa^`QJ>%OK(78?1UPSEgq z`7-&HBiUm8l$XX-(i>xR(6*+J9ZggZB`-m?73rV)@s07 zdF)i&J~(TqeC?%3(<2LJ7>5EoONZyY@$VxgZ%^C{Gnz&sn|nDUxyanl4P(@NQRm!F z3S@n2eq8iS+%}a#`T7I3(@x_j3bd)Kr1@Nj=RL9V9sJdjPA7LE_J+KxKiXhBB+&u; z^86)5Z{z5_Iqv6v#=@fp0=1)kE*O8R5HOW1tBIb@$hV@HxW1Voh%>52ud{eNj=xc|Lv{^o!P^xyr9(30}!1O+7cYI}CGHq;lB_l~lgwXkO4AMkK<;3+S z3WEYSsb0QKV6XI0yY)=|v}tpg_|)AAc276y19EcPuhg_2@wbq4UJZG2_1JZq<5fpk z*)-Aivqm=aW|5!ScE1QdoWyZ&sbeDj)q9=l)ONBR^#!rEa)+&(!$~*?hAywR@8W)2 z96cjiPv!Y&tnMs|rStCoeV8}YXYH?XSY!OQ2%N|%WRoBfC9x1e5y_Z*6-o8-w?6Z3 zxQ-{NZvSTk?Jcp1&(Ns(ZI|y13oASkCDIdSc4WAH!~pIYl40(~V(p`b%=*{ybeqzvVL( zi(7XyE4NxqT?;i(+VOOrS$Nk@_6VI0shhrI&QVjBIL4ayT9cLfGE9`xz7!QL5#V`G zpE=c&a_!((4+Y&U>2&*@9Q(dp0r$%X#~SIB zU%q{0|Hw*UUprmA(VczgQ`AH2*xN3W8xzmyjl8h+C={7}klwz!Tvf&UQJqoCJFHS- zE2~-`E2(^vwNh?r!{jZ2(@W%AkIGWTd?`J4_YO+zOstBV|7SM^Vq)A)|LrNteDB0i zvVw2UPf_(gEsSs%Ms?y$GWS-QMK&G`pA(|H+O;rfdr{y#miKlZW&O8EqR@R>>M$Or zyJ3u5<+jSddEB$Kokjy;UMz871!efA*jG;a6P%=5PLfd!GjVL+`>R`TMH4wO+Fvkb z+`7X$I@E9VPIshF=%+=Ou<5!u)-eU8%_BStQv61?WC|A*6?=D5E>@9uw zX=vYY%B!9NXs`Gcb+1@C4 zyLLKW?4kYqc6LTEw~meT_O9s1JJavc61Seas~U5Zq}8Q6aA`s!F7Wd=gi<>QBOW2b>^Vfdauryp{^XyW^DC5vROwX?j2JD zbzN(Ju30$QGxIAlUZnDtBiMZJgiPt@$ZO6MNAo^kd#xF@5RVc%k1Q5camWtQVXiHDuQrg8uXNGo;zN3D-0_Gf6dt5P6*O8OvX*0e8@4Bz ze;u_hz^d&(uJS%0OvxiZPxf&lDfh--n?aqujTjinuUr zC06?(lgq9Z&SxsJEW3S#lZ~kFb{SE}Omc2Fv8xMxL@}9kt`gTCl3b(X&9v-8d@rZQ z_W6fq<=BUQ#%f`*i{a1Z4+r17>i^|pP2~lVGnwS_IiauPI+_tvW`?9+uUbV~1<7)) z982@I%5r*V%Ovkv5$a~H@c4Am;`_vB%JtSuZ;n$SI^I0LP@iI`KiBbYic<4~%TQAu zd4Kk?yLK!}?ES_)j{NgSb{rF|&W)5=d`?z*)@E_XWbw+rft3#nFXzv!yq&grOzA+U zqslpeW0tvTEpxUiYmR<~#_UEq(O)@xzcQl!HD_-F4xKP?=pc;$%c1*~BK6auTT5FT zLozRLb1OSLbDZ5qsGca$(kl!T=0JfoH43w*a&)e7d7{rYI!>9X~z<%dV zqCn0XQc?k|Pc#ezkONIo0;X3Z!Cbq49jP!7djSIn*P%jEKNuh!MmSdd#Q-~doGl)f_8kIIhd$sWe6loAW^8_a`*qbpaBC^KcJ%g$09J8H5C}b7x9-h zg0*w}?=J%IC?WN*^)CSz1WZsMW=;4aek~vT`Fz_sSn}iSvF7~XPQv&B|JojF3I4%Z zIc(%+!U3lpL{@(w2OL&NJ&a)F@Lz-7{{%VcCcr?FM?Wqq9DD;n$0i*3Uk0>m=f>F- z3pN`8%@1@i$J(1Z;?@s6&e_`B%G4el|6l&)25tcyl3)fz-1WsEfRYUCAKYZ1ONH;% z`R^8k0911{g#O2u0r?4Nb|T?l=2w&Y=OYEk+iPteSOWu`InEY*8%yI2Y(JPj3dr%` z5du>Zks!B%1d>#c%sUD&yOD4L%BsMY;oJXBh78GoU_j5&Psku$vUY+9hK%2&T3eXg z@;ibnz;BLoG{xGR0KqB*b_4qi=_Vn7PRBavpMC(rpz&MRO$YtMAAkb~0JtB(9}onJ zm|(W%nr8ZMbNqp@I=+*!>4>4B1mM)7en5tSJ}Z9nd^m746qCg52w152A zWdt}Nfmd9MgRv91-ff~f(CB|270RrJsQ9S;RB7eG_%Cnf`<-i;$mov zwZqvsfN2Y+z|VCt#6g{p;5YDhBYP7Z)^H>476NFRp`aGMwNV9NQNToEfC~>T4oKkr zesUX3$oT)h7$oo&p+Vz67Xw^(6x6{*@M7>=%^!CFf#Np+BB@CIjTj%moe~B`fpxfR z4SXQ-ARO-hX)!kg?t1$kXp#iJ6g0;ddToG{i^dRgPVgH_XhT6KrxDi9-ooDcS5FG4 zc$r!n;cRi11{>**17=822sl&d?@NHpkPvA5C43F|P3Ptj>tJB$JZLZb11)d~(Gb&) z;2p647295|CYi|!dhgOz`wx&k*SOaA^KLG-ms5L2JLN5TwgWxkwf4t_KgcK{hx2nRP+(Wf+35xJ*EzblWir=xB=8%QA-;jw1Tmh)R`NuPY6b2?|&EykkHrH6BBOy=x2hi_g0)QD!@V?@=jjkYEhr|R><~Xb^H2=;7YinU;Y3hnI zw8dG2=Nr7~AaMdTZa3!*|KYHJWDg9S7#PT4L5y^0g&>Xo+i^0;=;6DH|Mh~_Bom;> zV|_`$-(3q*30)F?8+*#!1mtf`OfP`%^m`-f{yK|}1pIX&NQ&SGR6s5V5(MAR7a_Zk z!f!t}3o6j40X_WSN%;YF?dY#*5D<1yfG3IXpqx3SrYxhR0`2Jq7#K5wHWc2-=KeU? zAixKuD=5<*}a4R)J zm_r37-$17p9AZI`O#A(p55N7}e8kZB8&J;#BYGgnYtwHa<3cFrKaHFIvjYg+kTnS` zBijv}1}NPi1Q-$PH~|qt2uht1j1#{V+#HgX>AzVA5)H{s z{X9NU^ax#1g6|}L^S3tJ0%?b}Soe;8u`TO=+R&>4;$Nu27EH5%W@$kqwFRM^4hRoG z=>Gdp_TP-E{A{m00fZtA*t-#>wZ`DG|5DMBdFKh4ejvrV-7&c{Hn70Dp7pe@ec))eQ6 zvjC~$4IF5o2MOlZ|8bxpiA$)*nsDs+?c~3O9Z(`-r5Mlp%A_h{PuHgVjdcNij9opT2qq*58ygb6nMGUR4@tNas2V) zrU3mOeIcNe4!AJjfCKY_0FszOz{UOj>xkbD{sHp$#5(^v_XspOLCs|yB@7&27(h!n zPW)!@d)*WbyjXifQzIh>JCL*n1Xyb@*Uxa19Vn1-5(bjgKUV`%P8cxuk>J(*^&HJL zR_TAZ8W0W%1Ixd@8t93K{9D3TgWr1oU{#?Fjm)ujCL1vIa3P_!$bB6w0+>q-@LdT9 zi{D&s8d#9XS!)OQ2^I{-1DBF;toUu^nxBonh_i=IxV|k8YX-56fxQ7g3~;8q$D4h;cPtM4kE2wwqy zyJ-vAd2=WSXlQ6}i`$SV3kD4e1AfW+h9d!75bP6<7Qb=aM6}RYL<|tL`iT}q#xTIH zCmb$*6Y2E5Jc+i)T7zB&Fy7G~3)H!x%A}F0^9FYtB-=rw5ojL&aX8kCOoZddZ$1A_ z{68!|#PJ4n0bpUFDXj=Vl7*O$zkfIJn^Y5rAG>d81-M>TQ0~FOdhKb!I@n>&Ex=zk z`oe(s90p`ry zE7Y&qP>_}Q{>HC`(gs*S%rmuj*}zK=agU%W&A0dhi+Rvq+RU_e{_w@=fb z=4<{r{Qs9%08&Dr+Oxg@AnFck+u&{!ya4>h)^JUq&)D45a3gO#@OuTopl~p}8bSz0 zw8EgS9w1{6O=Aat|NX7TZ&Wu2DMVgJLppcsNTD1j08B7Y{I+vbc}Q^JZ$S-pKk;I~ zlL*cP;ehd*&&>r45xjwawGJ2o5?7!f;~xOVZ$N(p5}+s0%5ozQ1URJ#kiP#36&wPP z(f*qxIvk017*(Sp1V6 z#&2a^u?A*#fJF%f)leq^XgzTR{U#gOW&{Sv9zedq57bD&u0sBkUB+)|Hy<>ZbOl^P zV9C}&1CkzQ?a3kZV*Z~-*V4)!tB(cK`|JU~!Vy#}Tr9u$Z+vfQ!)=Jh0^Axv1%ld7 zAW|d@m@rT>f))i5N(8(v_|5ST&*5KP1Y~d_H+_8-psN4_9eYBT@h8E*KSxL)!o!){ znVUi^478zv!$vC4P;y2PoE;cwQG!IP*OD6`Mg{-Zz=8Pv<}lP>Yx%7usW%hvcMAjq zgXhp74h1YS7&z!KkRAY!`EMH_P_Mz~q9X{kVriSH-@y$}Q(&>j;A z6Th9@6qqn5n>&u40?xcQ*rNT_`Qv@!fR4{*Tt09TD*%=iuA_dXI!Xo}`VP|y4xY^-~< zKZA}9O@M?0Hx`dMs}PFzZ`6M* zOxNhLYaaDlSMdfm7oy@pS^z&c7lboH&;cj#C;+A{K4Y?}a6!a^0j}se+;yS>!Eo{0 zK0B3fjIuHUzB*zeRO&_}M{< z;@E5>uOg9%V$kaorly$hEMMD^LO%V(iza2FO z$vT*jrIqEM`_eaX#3AMz2-~5m8$_M}J-Z-}1p_|-bB};Cj^Ai*K4g%;1*Zc26Ei3- zqap8wV9>wX$iEw31LlhVd0y^gmIm8*=+E)FQ~au(mj~z%U=AN3ngPd6X-9Q-!mLhK&@)CW5tOe1IU<*Z=BQM<^8g1M7w zUnSp-QD4tGZ870cs29W1JE{{kiw?JV68CL*=E#Y90WoZUz}XYC?``?*(yDs&KMKno zoOFq}FHap+5_E2!RdT*YV!lGfe#Wgq5uI&``}Zfh3)c5F`U zxT8RWqn1<92g6b0eKv1fIOUG%&r0uCc%phrgL84A^aZ0gN&IpuGqcMLg{*C^mZ@4# zmB#JWc4wxvxfQkroc862_dgt$)l>N9&G=LsT~vT&f2yP=>7aLT2wY0T<)P__)(e_} zk0O-pZ>joCL@ey@jELpNM#aCr<+(dkndQ}IQ5oHc%0BCA>q_rtny!6$eN?9;S*er< z-0zg+7Bi7IKgga`Jtflo{OHr47iU=qwO$x8#oy}JARtVXodPV6wFYB4>kL^E{d zhBxBk>i{cvg{KASh33xYeUTqJu2Pp3_>s&#!ni1+$Sij)oj&$B$dpUx2=TGq-YMO> z1jPq=Bs;~#t%M{zoPPXuKjYc?IZPar}BmKv>Lhn!Bj(e zL<$9WPx=UGq@z@#(v}`oOz&`%3!aS{BFBif7NTIT6`7Ad((Y3~)09UNC;#c}J~|CQ z7z>Ahbn#Mmw%;Qhi2U z{V1*AwhCWMhM*V{MqMQGOWM{r-&_{#F%MAHs8 zwKY*Mlv7<=K6w1t!)iOrbA@@xBFSpaVz%oL@e`~Qdce? zwEnch@_HxgI_|Dc0h#Rm=6g~M-Tt=@Ua_o9;7QPK`V{D9OY36EdUfxsK$iArk&(_w z9aL76)Xj(LfjRa0iGkH!!QR8!)I0AC7Ej<|66V)v!Bt z_+l`62g+>A`B$#`^E}8#BR#E;!Pb)5YB*J?XSc{bKgL+^YT9$uUgnc~3wuU3bth54 zmr%6|vY~!AnLYE%H>45bg~q%F$L*7Eh+*csGoOzRcI0`YZWd#lvJLoh;rrsYsm-6XV09V5s zByKn>qBAL+ssMQ?!*b*+j+0N*?_C=wYsl^aXO7CDIi2o7t!rc~=-zuLzoh9C(Tpg^ z9G-en&U$V?e;nfiZjZ4-h)yOK+ zURP7xf1z{b!BaXx60Xi!;lNu19NKrBc4$q-4!6wZmu7LrQ)#`GN}lt${N~zet((uj zYK_j^Npia1^5uXv{jSdvBXLEV(_F6_%Q32B=TaxqqE}gI_?S&2-bVRn3Nedxu?TwC zNb|gnEPOsGi?ZG-Xd*^2t9k4Ah}8)zq}x~K!|bVQ@T!(qK^fw>yx~WZHbdmJfss#r z;?Lb@6Pn>UO|j*bw!v}7l}-W1G4?6!iERD{`j1nx%06vVm%PR^++IsA_c$w>}Zs-CsC^qY*a}sT;NpX)m@$`He=ro5s;mNkykdWu;+C zme|hhDfVR8vA+tjMJ`xWs^4Up-}2KZE-4F6dZD_V&r6>mRzBCWIPtQp9jr^R5>n2w z9E&tRYAuVgopbtlWLIt)X>^9u6nn^fmi@;a-UUAvl+U_nNVQuaxRRdL?QNUJ<>M>Y ztJpO}eZ9Eowns^FUP*8{{` zh|}KL-0O&Ztt%S5NJl?6FoO0tT{32PgQ?Etxx-;!zizB+WusGa@W)p}3TH>@6BH1q zyg40@iI=}gB&Hoy?KUbiuUKR|#_qDOW@eF1X~|)6{&R2OL$VJYA71!XXWgJ}+2@;Y z?_xJcXD*Pn4?f+!U~uFcWq5THXImY+zTazy>lasdCIvUd%EVOi`#e-R5Q?DxY#Nx% z#9AA8R?6PQpgF0!xzm9=_JiDw!N!vzYVL>BrEY7tl$4Sjq?@8vYbs-wOBD)+HU@Bl_Op(NTP z6K?03lP6%-a4kaj-Kgv1ysaWT!dFhp<%~w=j7A^udF%ATXsS6l3H!mWtGyr1#l2&! zfm@SAkm*AYPu-g(*nWmXmN#tZ4pJE1b?EVdNlfl#FiX~8c6V`w>Bk$L>76cbN>`kn zI$u91RTksZM(5#5l@>pIgIW>Zj4(`=SZQ>Nk|@TW?Ix-gqzv$G6;IAPwd9&;&P;wM zR}IU1erZcs$woWQ`ti_#z!Vut8yp3R&vy4OgDUQIrgj zeB(zUW(!}tH0d%uQj^+KT$w|{Bl;xKoV&=>n2+`Kx6;r9^X~VJD~+-S(yTx3QT40m zXWf;}aYbjKz^kfR?M9d#qhXv?4ej%IjHqs#)g7X(pSlDE&XxD>7@}&-VtGn)KGPB< zVK`0oTG-*%J`wBk12~o?dG;^a3{p3C3m)n&W;qsAKJrDlZ@bS5(=~3>OyWxxXQ^Sd znNdQY9_z#eed*?1`5W?o3fSYRBO*(qU}k=Ceh!nY`yEBeZ8D0 zCSmFdTZX5#5N@yJ6_zC#ukp5vjH-##1sz(MmdUZ|!)bDdgl(?m9@X~8?D4AW)qCS* zn6}eLFU^_?$wZA5i{lpETDU~6cj_43C#jN55fW#C@b+o>Gp(W$do*DgiTljc8{?mB zuWW513u-Uq%wy5W-_^npB6qo*r)LHBiKtShS%C5A!z~Px{zgo@o@Vy#y>@SW^-zVM zq336dn+Dg)<&zX*(c>x^JKih`H40oRx7(i}mh7NA8V9d9znH{TWq_$=;hiu3tk+lO zRP8=o+LOqy)%hN_-$u{WyuC9i-(S^A@#(wr#CmcYgRtTdzCsT}qw7u_ORn-?+(r)u^RTLV#>OV@==5nT2s^)yV zWNpMV+ijXXNd;JDvFO70JcSQFjGlbD=L^*(mmVehk1~p1=?rf4D9_9vDpIjU9op6{ zB6+<2&4KBMXUk(6ZUry|rxMG!A4P8s;H_1=%BQNVuxeaR%-Hv0DpKjoW9wCh$0?fU ziMwW#B)>|dy*dsBzOABePX6+NT(9N)qImJi@=6!pAe@sFHBGL`H|7hi=*JP=ZW?WSm!?VB55k7;0ZJ5rRFV}XIc4e-gY7N|WYFuSy)ueYe(yvK3dTZXFgX(d?a z7xAe>-9Zbg8to0qk7`otG5HrcG<7qY5-)!HXd4#e^49bW!{xs51GUY(-JZQy>X}IR z9{MP8&oeFtCm88GY!dQNaGyu=wKH|u?YuaS8BJrFy773c!OXQTZOrTW%XKR!=mcAf z_zQ2jxgJ~wgN?OwHPF;fLr-$q%cWlFY0R?lGTr)eyS+lmh^xqk^NT@LRgtClrDOG% zhTBf&<=B)o2xm21;m({d zMiqeJOF(23(py6PjUMXehs_YrLl6*bF@jK%0tlgk1__W%!2m`S3@B{=c0TKG^iVe+ z^!E%L8c?6mfKc|MoQ46tODMp_|9!~#bBf=)^n?WY&9Ff4-xemc5m`A zybyrw@DnvC{vm(>1L3Ign?49NsH7qEaV|jZb^~uR0@yvk(^$uf1YAW7C?*h&6~Ea7 zjXIzY9iWBT+1ol8fZ{c7BZ!d*z#al(IcUHzfhHXxg#kkiI2b_FVoiYKw;wq`#lu(J z+ib|-1VDPP>yW>@Z(s=oL&k6TfNG|zqbX=N#$Ci3u3tl7wt=pW*#^$X4<`=vPa#(DB>CHBt%uXRG&y*rBks zwt4U(|G*15NB?ZA@ms^q#0z;wzwR#_s44xk{l#wwv9`9RR_2i87HVy**9Oip5^xSN zfKIW#y`W+W#Nohp6MB;WMtO8I;Q~f86n^}K3;O%vKuMHvxcDs})ED~mp%4Q0+QO87 z(`wVuumUvj?bjEu-kL!80`MEbe{}&r9zGaoM*vmzwdfUE4PaY>8PC6MW59!h&&K?# z=z+Q*6ab)qz=uW}Kw?#d5i~tuD4P8)z zw;aEX+-%6;l0v89CuHc70>U5RknsnH=BD~qj;01?F8n4qa|39o9!!WICco0 zT#z4-bvCfi;10rojw<>GVkkIALr*xtkntPH&4vtv(i=dB`zL0Qyb}f_ID#?%jV$Sp z1Ogh)|Gg&&zLB09h_xjM2!88;;m{QX99+Ty?p3ttYLxz;$AQex- zA>+4$KX(~O|_ zJ;8v0b{#U*Bq0RNmm?VR|2Bat97n(;0*)i9F4j0ON5lY{w(TqgGR)^hck(I53#c@+ZF-?6%7e|b#x|BRO2UJQEx3V?F*>ao%hY$K6*hq=B zUjUaD`bC=K2>9iNFhO^7!3S`flM&;^Y_2PuO{+u>7bAl zBdFqmOcV}lvCa8)w)Q!vo?=?hFS)JfKHBjG5KVsk_;Gru$n>`5OUH*j3X4Rc_x(_( zF1m!i`?lDu7Z4%YaFum3NMR7ZN6+ZVSMmtVO~tbMNAw}9tJWT$2eY?_UzvK-Ay@N$ z(TGz=@j&aCLiK_9H?4|?ZFZJ!ofkNq5T|yhw6<}Z$I-}>BYasp7~@%;-Up@Cb>&)} z>70(4WexJe>1ml}8oE8Lt&VrJcj-N8ty0JylG80puQXlU#cL=tV`t}^J5|?KL|XSc zzK}=$j0*FzRCC;-Bp?W6Q zpWDGVgk=7df|HtuoFY}>%h{CZlt!JIvZ$I2(T5}s?bdab_sc5z6tc8cA}2l~FDnnH zhoyc97&7dZ@y)oSmd5&iM9fVt$%HPb`kG3@{LAu+-L9p|jvg0;o{!$-m!}#Op_3CU z$f=>&dOI`DjLpkf!LO+}$wJ&SPaU=P)l&{qx@oE8k96bC`+h#^wbkv~{sI~h~>MVP9rfHAr>1~M?J{WZvKG7IBmG){Kx?Mj0x#noRjU>2{L=Ht~RdT zd8$XEG5R|L;>6HPcdjI>7N2E)ktcn*Bk$hRW101;X_a5o;n&1SoQs%KKg%$OMoelh z_%71#Qm9kuAE{vq@VDyS85nA?dz4C;C|w!)EX*2dD2FRjn?~mhC4zWmajowOep8yv*ol_vaEb?i3Fn^}E5wti_=zpts(5AW`jA#^&sW8K;b+nD42m{L&G*IJl-JReNG&sTC-M+=?ds^@9K1ttx-tP|^c=$o>G3Q)! zkE&6H-&59I=CsUOy7noc=1CC|gn0sF&$PQcnO6Xu-wE8b#^lO6yR8h>_TqR@RBo`y zRrV^%E6<^jF_8v0lGczO#Pw#Zaj@iGWv`k$*9j<-S|ku5y`hOIP*NrH+LO2c5O^8kN*Gm%3IUr*ShzxUxwxcQN&tzDY+WphnRoN;O_ z!rIDS8u2cC@PoF}!l;?X(w;qXmM$+|AL>+2nmXmIGGGG*iB9CH<)CPn-8Bf2hT}ae zBop8!N~_u{ccQjP!IJNV%1Ka^U+(2tM}bJP6{MtuVnf5MXrx}daHeb>=3W*dUhS0% zsyC8R^`;(Of;|%13ig}8*ZVj70%y+$u2oYmsJ5cT)?xg*Kb2BPq+v@gifOZ5(eV-; zw;)S2#<`4?>4WWDXqX(Ay9DYa-yDF6Vt=tF&p*au4oc>a;K#6>#l@YnB9zL z_&pSBf!aU&V09*0x#vP^GAmCbLNI8!mO2*Lr8u+1<(d;0Fl6pLFEvj>h~Jf)iPJfV zQD`xT5%26H>&bi$A3fg1OB*IMG%Tw3>=F=)>vIhlqY@WVz4K53U$)62BPx(^&I;l> zVH@u<1x2aA?JLg`wxPKq8{Ma_n5L~Y=^GqqGCB^A*GuslS!0S>b9It6u&&j<#4;q7 zf)2IK6+SXOjtrwmeyCnYc@;VgRd`F zo|wpR;{Ea*T|c@eEHDZq;!{7mYzW7J_V#Y8;;YMn+T{3onyPX#hE_N0epE#|wPnxM zM5{U>FZ&Va!+lk`C?ZxRbzm?^$S{g$e{z`om)B@0>Q|QVM31F%-Nk0+3&uof<#mZE z*3s-g{F!>la6YsIOoOO6mS^JIPs4l!r+51CqyZ^pYA~E7Qxwk2oqc@^bYn3gP zzGPqViQC`A)_i(O6J-5hDorb#-m9`Zp8ZioxvzBwFUvx2E8Gl9bZ^hA=Mlw6AItV} z$~jl=041nph)pklh*da**&hY;HSKa}Q!qQmFeouhTwd}~ zvC{1D)|M?#?R?uC39%ntvOKCTt!E%Zjmh-E0Z>N;8}hY!}b}?X`Q^e{swU<*1;y^gsH|RCQ>>Hy8?Q^=pexBu^#SaSvEbW zeb;F)xwI-s)R!R>cpfO~=UwdnMi&p}DYdHo(9CRjrY$QqF0wY4JpDW&#Z_XUOfqst zXQD_!mx8aVd_0)v=$gI!63qZ5P~w(>S^&r8aS=a1KzdhSKxC&Xulfsf(6X-5SDqolShY?>WFwXKLXc%wX#N+!8b|E@>Jv06Ql%LVa&KI?! zvKZb`<5TVPF&p$#-woj-zZB=IHnsJAeP~nijF(Gf5(&J9Vv)HneRvG}`a|v+KKf0% zfh>>c0J;mSF1>er(#Km(=1-}GL1*lS(i};#_=2R&#mPj(um^j!>I>imAq&=fz7hy# zdOG>&lQT5QW`25+h}hVwN>uf9m|Qk(0&LqAbvopBnH4w$^a4X-+0(%_8^JB9rSC=6 zphbvz>RB5<>|6rFBxe5nF-Xm?XOOWI&`%vR%Pkz0FpUO|Ynqb%X!ARaUN>J8D~@T1 z@sUhI?CF;UL(i>Z8!cflQ03x{*d8+)1)^yMm>DgW5Bab`*}!8C8Use^4!q? zPTWdlITM+ScajL(AHj|=> zotub8q{4UqVLu6Z|Mt)4i8W$U^_JQeA3=BEQm__ll=4!JqjetM6B(@a;m?K}^ql58 ze%pa*vUTz$ha%`a%D1>1YeFS3X01*{xZ&HFNIEO9IsnnOaNB_Ji{3d`X{*0XufkTa z;cA+LuGgY;A#O)z30#TXSM znrPYtI}dV#wG-@uoJ3yJ)b!BhG)#YMd^c8kAG(nnVBeDj`sJ;ILtopJfZjnlL>r~U z!(@ovLnyjSQp8>6s~+F=*yA+tF4|L3cdIH$=uzmz!Ps+S(Dxditcy<>3;a&BhTqXmx*%bq-8$89iOThId237;u^QgYy$C;Br}KGmxY^<{7=4<> zeNub5v%rFGj|)!}n&~Q{oaYyPazb8J_Jnh!%bM7J2~Tvqca(8x`VJ-H{hI09ye?A9 zoU2rj%*2XEs61?Vb&^sXbxR}P?Ygif3$p$=Zi5V%Cb>q2$6Y4-KsK^a2XsVxHF4sr zP-2v`@PX-pF?VmUNdoXWwN+L;6_TPSjGIUQkB}!P5T*yH@b_gk^++{G2{3A$5pf2> z5GPdM{hM-Ygh`qQnJv_&LG0bHP~5BuV@(c08Yux*TA6iX z9CuhosNT=tOTfnaLTjPa`Jetw=?Py9yT!wB}SL=<@+G3P;7^{^fTg#>ZH=~T{rMOO1< z$%Nt<8ii^?ey=;(ZDpJwuQy|E3AnL)Px$8TcJo4DTMAyJNzaADJyW2*of||(vIfh8 zw5w?jRh^cZ<5?6TS=>;!Mi+kjo_7#UK=9{A1*u#Ly;wrQ6@)fQzmNYI_Y*|iki0dv zPw9GhW%zeA)MIn6Y-KGbspZUg5roAd{u(w*7Np7C*eVDN5cGrZiEjITQAIVSHCZ%I zZlmq=^!SaRy-wMBLSkYlc$&YFQ!E;oOw<)`ZH8k_OD5K8<9Kc-Gt(6oTO(|S}#jIxEo){CRvMWx$HZVDES?%&s2<3HB1~>BC*s#AuEaXly_c33cDirG*}Krhl0$cW|VCv9X~ub+oel^$rCi10Yd+k%0o)Brkd9FDYm*d>}9_5B&Vsv&C3} zzx@q-|ETFAs&8s;X7A=CDfSzXUWB55h^PS5tUsbSUz}qBItXZ;GXSPYf4S-Mck(m- zLdpISBu7^fS9K-8^~2@YA+Z271q1N!u)S0Xpj`fqYz80!j&Oe+(tmC9{y&2BN1`FX z$pHaLoGL}z1fLa(RqK+nVq zI9&tw7;J!C;#WK7EC3J34rrzU$5KFj#rdz*0lo)VqF-<}`8&BEf8k*G=K=X!J(&N< z-~-C>@|gVt6<|8~*VhBrNx|^7bVGMKmMzpP^g5P&fj)IHngtDl82I!v4`VIjT*B64GUW`gvq(O~5EWqw=sDFwR(+uwR_T~hSbH74VOi0>NTNho<$D9b5n2axMnmOx@I@3g1{wXpk+C5>lqH8FW%=UQj^ zVOyFSmqo>a(d=0MW`+fU<-}Lhn zTOxcDk=50w>9G<};a$*0AA{0^Ex>6ISO&|{G-jiO8nvt!sFOOL>_6OAVK1aj6@|%_N;*g9 zX3nNVF0Oy)Grpos`*DX-X36}xV)?*NU`_6Av>DxB9nD`zDVTmzmNUiWH$>IK)+pRH zNx?xiRPs%JTB6J!$B*tQRYG2F1+|k86FGlyWZ2i>s~zeC;$fPddo|n>YwoTI#Emd> zVhnw^8pIS|vLT|L?ffgfcEr~n-+j)>>ZiZB;8sx}cB$B%v9?lvEU3Lm!{6@R0Y#$0 z=;Ua|VX8>gbr$T#Vy@`UvKN4!sY}B$6-YM={8V4(RpqqB|6b9bEHLzHS?9j!ECKUA zElM`8ACYVJ?Uko1b-|HESeGIw4wWQZ(UOWA=N$7V>(+A#Y^h}_4@gZVvep}~WXlF- z$^w(@yArsf7@LMu1eUT8cRjhR>5D6i;5in>{Qwgq#D37IarnDru%;<(gaVF;No{Nv z3f5MwA57y1 zu>>aBy?DFNoJhVWN{_9y08Jg2yAO@_!i5Dlc_}Q`oR+V-mX%c0XjfQI_A(SRVTUs$ zaj}4zpUjO-#6cIH#41~#Ru8r`sbK4wbJQEI0?h^PbzLmSB(l?0A73em`*k(fCC%Bf z>3PU$Ihw=f+D9pH9Ere6UGj`F^vV)@a2&9%{1cqNI=dXbjaN zCLG-y=tQwh^cfX`>FC+Q==hbZ6T=<`2N47pxc3^~vYQQ7d> zIrU@LO-L`+ey8#+y%hUtm5Z!W^%{~&o%PIOYUWyrknbuhCG}IO=FQK80-S>!c5Hjg z%(s}2pQIUY;-aK?9--vb(Z^#HCx^+s*w^;q(!zV3^EyS0DU4-IEwb!H)7x|Xg*dz% zMr9oM1x=uS_`i~wX_z!iq8fyqvxA}&zI5`x)i*1!V{xHDVYYNm6jZgU=k-A_bgE<- z9G}1*MQ5|@{{ALSm-4;@vEC$Z4Wggvdn*daC%&eocRaeUJ{>4zT8+;kEV_RtaUVyl z^A!WS+LZ5q9y)@vQO40NWm+j(*Ic|0M%6sd%un|}{u+4X57I5ymhRM((9TEe!Q;7so2U!-wM06(1t$)ejrK zm|EgK5yR6r(ClN%R$r%^%u!J?t4VhE5{#mvUr&W8JjZ9Fqv(m5VSQv5zYLQ1X-{XE zVU3_i{7KR`MF8Js?5&fIt94Kj25l*YJrrw=Tddn1E9{Qe>ZC$VreIzLMRkLfL@HDv z<>%>DXA!fWlTw$+LkuWzzq90au4bAJP>+3_y8K1`BZMmRz{fTgMgAPOAdUeQ^-X9# zBRbC#2HeYp#k$P!9rU(*lvj#AO>YTc+F+scdiBSz7tVd(v`pZgA~Z{+ub*O48g9Nw;!77#dc_ z3+_k}89hmOPR}%N<1KjW-5JQT9cFerF8tFwE{wY!J5{a~Yd1x?RnC4`(o%WqrK?f7 z0f=F%_hYExHFD^aMrdi-)Z1wMATz7woD|nT$Ur=5j_`kW$U-!k3oeqU5qW=Ja4;}>EcbAaLM!ya}17GRMbU+PV10LRQ482uD0YFqg`ulxQHx?Oh*1dkhFVbEV0V7 z6(8sDsaEl|F-uh=E!LO-SA+k(Rc6>(CvuhsVyRi&Y_i6i_W6Uh%#CHRw6DVhiZjcT z!JpX4*pcJ%gC^poi9cSZL#1K4MrzlXM&F90NEcy~q~;VkXa*0gzT)#AuK|DC;J%sD zwvrBY@wj+hy~_OYiFyoQmF!LqirZQgrEac?tb!lI$6LUX1u7>Vn%kXWk~7k6X+1%g zucm{AH|dk3dlkvY#)HyC*hu-|1osI$0h)|ns9BBtRc7;vU?|l5Q_=JoDuyXL0`wh^ z{X;itzZ@g8MWfsXcZtR@^d!$#%cE0;#JZJ)hblcvhGATLum|zMN(x{)!>)YFFkIahRXFVR zmFZHd;-hdgq99q$jF zt+)&g=>}c6EW>$6?B`6`&XOy^wor2lD^4GrF@jE89tB2+&Ne4n3VRUT^7ZDrg40L3Ed1FzHMi6C;OnzCH&kfn zTeVoevz6{GT{YA^i?epKYld&C?yxaEB$2VtF%QIS>4O<(EAw~}kyr7HGkZSoMLV{m z>OuQ3aQ&8BnR@Uft-sInAietGZ@-pPQ%aR8r2EN{vc2x}CF8DXmDizKL>?h^N!_fB zNGER2%pm()T#0805x9xa9G=7J&0f|`PtI;W{QF81X_?ilrKfEsQq&NISSLzwleETu zh==Y)f3P<4idDQiq{@#WrJQw=4~nbXFd78{C&=&&>$rw;d$E=ebVIWfs39F(|0Xlvaj&azPMFy0Y%uHROgL;nzrdY7n{L4jUm-e@jtp&mEAV znh?U})WC9*lWc-CIyh1Oq@s)&2bx6#8jcSECVbyx1v6wlr{X`Rq0H7tOdnYKHQrPP zH-9_KqQhTQga@IBW*8OS38g)E8_c`h7;E*;JDMo#j*#82J=P2z(kp9qNIpA0V5Awm z7jn?Rvv;ggCt`WNt!k|$!9M|~3OIwIGwF-xMlwv5QuX!ATJIt+4N?>S5m9(eeEFon zr@)CMmd(ZCWE*i%>}vdGuOVxK{YUzWp5Hw={tid3naPM_)Y}55W1fL<|Dfz*36|@| z5|IzG0SzqG?^U!L=Ig6OB9L^fyy!j@{}cuZsBWa5UN|vF%%BKn6%ZMMXODrGaMp+M6La0hOVk_ zv-SJx#TW%zeoD*v#}7(6fd(_?StuLka;V7CXX8-?&B@_KQ;3adi@Vrrpf;kt?qOW| z-4C1k26Z7^oYp)>*k6YcZl;Mm{n0G4IXE|#OrCr9QbDk#}F|!)Z!tD zi?R+)>a}7#ofj zb)K0T@bY>4cMsSa&nu(^)!)ZaIHL8fP92!Xf?eom##;&aU#sGBlu`ILH7L$`F5bv6 z8+KacfmMv1WFqvMn1pxSQ&Mqfa!BW&x)^F>1qB|#@^s)=48poM#%{?m=?2b6|afD+WahLL|cle zJ1P)?NJOVt!ia+WqA$O}83N>@QW%O-x-?*g+(!pl1eYfoi_$2~LBdmYi&00yy1Rs! zD!P#>gNvEYw6AfXD6$}GB$u)pQkh&&B;XIEgR1gBM5 zyio2!bgJ=2fFP;T1UUPbWEsXO@)IL&T6TvMeRD8+M)UeoR+yjB!?Kc(@MQ0~g0R2W zB}JLukUKUFYIJ-&!LcNQ5t&9(NAIR+g7sN9P*kSZ4ggGGsQf9cz43+a&p2_U!_qE# zCw_KRscSS`Bu5Qm(ET9T%2r&P!epThtMpdKow8DlZ0X_bKwRTwYrLi22LVl1kI;A$ z2j9;~eI24CqIGEI?Z+@g7iE_|B<`nLJxJ6#61PM7gH=RNUd>gjo4Xrkpjm z%`q3pJNhQALu1Buh_uhRH3%Oc-r=!JYw^zYBT??mlt@s3i^cR-I()^mDfT;AYQA@J zvncrf{FpnC%0}penyz|Ta9)H@yFZ_$l}Kn|)v75t{c-hjE^oxJ^^vQ@tq*I>S{x~8 z>W8#5xqd8^k7s5QA1JZB+bg7#+&QCJ3&SQB)q~=Zqw*A)K=T#e)89%*;Q2F?Lnk8cF1dIOA zk7^1uItrZFk#Rq-X5fQ`c?B>r_(tI$9_MusFSXDb403wF7P%B8WAhVA%OOhPi|EY` z@gKFVLu(AAAt-fry{C6HE8{g8NwLy1t6Jy)5`5Ncy07EYHYKCGbl-ZlE5nyNj9rcC zJ&oCX1?5_`O=A`_`xF1gU9t?K7JU*L^HU4S@pUgzt(&N#Z=NBgeubwK_2!R*l>~dd z=Reu%C(WhacWGKCJP0uk=o>iolSrURrav*ig@u5wjDMCEu5!P`10Ar2cwGOae6~P^ zi4FNdn7q75MfD-23yhFE+x$As`COyp{u%tFtcvLGP-B2t`QO?vzd?-w9iTrl6%+{R zMgIw87(h7v+rRTKh6cahoO_Y#Gjam{r@!9T`jhnnxRgJ#l3t?uSb#P)+lvJ;&>Q~C zJz&6d;x~Z&gB~#8kU{^4L&k4*wqGVz83BKZ7s}xe;q*(&Fwkve1&A+T0sM7Pzr{fO z15hvhdH@uQ!Eb_M0ZPO9hbkP1Kmf)A{%qH?z8LKSCIo*S)jyks_}`%Vvs({zwOMu5fpvp6h3 zN-ZGc|ApfGGiZnZ1&}}M0fddP0z7m8 z{Xw3*q-Suju`<&;G13D=UMzHaz(6FBDQNOC3i#`%+#e?WKotNY25=Sm*W$cTE5AfS z{GG(q|Cg}-WLAI}TOb(@7#jH_k{7_p_A4j%-w9FuA3(hfi5LS(_s%x<77lcJX7s?w zz6m{$Qek9BYYTY$+gQ`u0)woyj?Ol;rZ&#B1|JReXie<(Y)u`2fmR^y$%xii&%yD} zzcm1oeQALy6Iw$fBU@TaBRzY-+X6@u|5b{H@g=zGB{-b%Ut(qd+=I-(6f8ie|Mfix zynDZKnEe0n9r@b=VFG;P*?@;SGbd5W#|T&tNV>N1lY5B*1&227KiH} zyIL49DPaU`0_cCewm{$y9Rn~c2RL#8tSI3NMa>Q6Y%l*1zi9Bw({Tcr9ikk>wn}+H8q^48mhduRX28NOlW`Z(1o)@VMD(#m{ zu`1k3Y%6TAx-op+87RKQpmjE2E!oEPtAZtu-orbZ_otp)yWg%JaNS zwsQB{fV}FX;lZ`Rs4`~glT(T7Ok4AZz@yEGqc<&0MP-S~mzza}8Mx2e>-&f56fSh7 zd{qh?3ZV?}1kve(Ddn3#v8U(~gq4l=e-yP*sVd(ndp4J0Uf`cpw3Ly3?y^i=o3U13_{{Fa30b?E5e?PgO;gm8IExOp8(e(8wzNa(Z}Js;oQF)@C! zvgo$LJTbNAXasMHChy%j|G^FzyO|tX?%iIqHlA6c6rC+6H<$NCxwbl?`TYFI96iI; zb;hKzf-=_SgS~4a^!uc$vv2l1!{I_StAN3>KaEe)rf|u((l}rLt@EWsE+hq_%{;2v zz(&t)OvrQxhPbY}QN(TNA5e_Fnu&+gHBCe8{X$)hj1fYEr%ZA0-pR>+jB&^@j^C}a z?s>=wc2?g|h*;MO)f^yM&KrPtSJMtn$*yf0xu3lC&>D4Xs29-=(MP`y{FbgHL9+9e zbJ+8(gSB$veZ6vR7gO_48wwHq1@=)OL*wL@FJ=)6+x93zxsoq(;WWo6W2)#m^oL^M z$b*FweT)y=b=Y}E5L4FjBogUtjg%XDGl)ZtiECEAI&?TmI7un^;BOlTlGhD_2qCri zVuL`mh^*{vVT7dz(%*VmWAc_rLC%a&%w$(WNZt9Uyt73bcZ0Eq!zGI5&0EE;Q-+$0+optz3Qdmk8J~8w9+@M5b(i7(wF*8Su{vd zAU(sQAW!-!kMs}x8LZWp%3}&Mnp^s5nBNz-n55w`8X1H6!BHohB}4ds9BVOVa68P+ zz0x2XXx4udm;JNe=583#WPE;O0-+u2)sSUotsx!OMk-m1c*SK_{x|letw>S=Z3$ zv4P21n&zPW7KFs{xC8n*9=fzh=WIBoqyDDwh$xwgx&hm?j^Xb#Zf6j$MnPZ6WVHDq z(;+rPB9XH*#b_$W?Y3m-?%rp17Pnht7Ps`Lq0hL7_UTi6z<)?cLh)4lgvOdHfG86h zD1CyP7%qD$orw{{V92&Pa^FVw8f`=w4))CRM(^0SK)tkSSf~Go#?-m;7`83}sWy)> z0z+Skc(y=s6rvh9m{N|(T_#K~Bu#!StUWY#_eU&m9Bfyg_K@~wqJohGm?Cmhy@YYf z;~Yh!YBNvHbWh~jX_KaNECYQ7)pKRjmLiktH>)+tnI$Ro(>ygYDYr#;n9Zx)N;{6~ zBjnTT;{n;&{PDT8kRn`=*kF>fkY&%0YcxoSZZvcTv&i&}ktu?#A!&?TUr%LAIBc6g zXwhe;Wh8{#TuLv~(8&(9Cw{ecMj%rt%3*eO=IhP;@$|#U+QjLUvVjULlh4*yp7;6_ zn6ja_?z=PFP<12q1#K|o+l=z{SLGHUmp39_h_7^R`N=moTi!~&*=Ek#HhQ2+@?@DS zX^hGY;vw*6?02S^Eha1R9Xc^-EU_uNX&fA=$x!u8pUz36m+dn7mOOWhmh>7iWLJ#i zJOh?-4_5?*Fq+PNznc(2H#g&RHYEZI7D06BiDH%~4|#xQ#IVm(=i-@N<_Btfy4z2B z718gaA_P%b&2Ej+B^Y5~cO69YPe8B+E~ub4VRT95#0A6@a-;W=a*%O9p~-)b{JGFk zk46@Hs%I-a#X=irHR|5^siwksK1#n;fdBKwG{jgSH|_wTc~gwmj8SFERYQ%f1V;vb zXS2Ql`H2vkI5w>7>k~6UY4oC@d+LIfUJw2nhVcgd_ydadmio9VM^?mcuJs5w%)*g>{bIJ&#mk&_(#J z*Gh*;)=-mV%o@zxpE`?~POD)Dk<5&v)zgowEx)CC3o*|gj<^pWxnt}& z`P`c$duvD4y?e_@iWX@8AiL1n{77EGqdXGYgYOVJXnTon=+`PjM(5CIo;hT2ZK@Z} z>jE8#3Q>AGols24Y`0A8;3t6vunXxA$kSDD6#7g{TCIa}bb=S9%b1P^BV9q9U`xCb ztmR4Y$^9lGzL1j0jR%hl)yUD^r1U#x*D`N9p#oVeQeqZRLj3+MxQa70shKLL*Nw%J+)5 zPrY#<(HTMf)m!@mHJWrL_!P+pMIWS7VU|>+Z!~1crb|8q4Ab7p)j3IX6b634n*Q?k z6Je+#Dr@$wP=MF=X2}DSR{3~dV`qioj9>rUnl!Wjx|5~!l?$@)q{5xZk=SXURNbH# z=uCb;4=*KT-$p0J(tTuyX$yUq=dfqX?LI<@4gSW{`DEm|L}QY2v9jeAf&aWWnR(@y z>@?Wwvy|&cJoiIrH#kik5MD%T0Sj1D)XDq+RgG_A>=DmTs2M&04Ism&g_OaTOjJOqag^<$X)ptS@R;=3<>D{`S-A z85%1&5Ma~FSMEcQt;QJk*adP#TAB@!nOIRJminoit$%gN?9OJzBx^jxpH$-0GejzQraVJE@v~>ONp|!l z1*iffLYWFO$+PeX$7hP?(k0*JY0aqkrI{m!ov0k_E(=W%>R1iF2kYc;+R6noy=)Il zwEJ$CNbpu>v*w2tP|J9krc!6HCbrb4!#)4pc`!Cjd+X#gGX?8Y>30V3-)t!xpbgm` z;CyJl_{NWUpLO8oh;OMtn{DH5{IcR zM4xh^Fd}LC7NL@gVPA|tgYhH$2V~1uKWJPV|5lgb>F9#-2kpdSk=Hh0w-q_9wneu*G#XW1{jWuE2zB25z8Qjbrszw3w<`l*gCMJaVs z>0NSf_dEuMoa0fVsYTHkZ+d_sji2{g&X*)ka*fVT&Ux@@O|fs+7)IFmwc$+1t)}z?o_u0hbKowqLn9ye1U+38j?J8Hd&AC498uEI#K!;NsuVn_K zx)x6HxVx#-wIwiT-eCH&ec)$N-E5f*YOg-#BST5`4jZuoJWUK_Vok=*;$}Z7gqR9v zb5XaehHoa`-AQy*(Qj?8R19K55b5Dg*L*SdE5|+NuM#U6$yAnNCUEiApl{bfS20!5|43 zEaW_)=tMu#0{G>RfLL_ediQ9MvO?JvayfB|F*0){(b?DCooANwA#YcMmb@X}DRk2`y&UXkx(j*!NFGWAOi%i za1{a#7IOLR;h;eE5>tlI9SY#3$)Yfk-Pkkdl|=1@y*q#M^+x) zK>;j*=+aP{bixH-?2;SVb{f%lK?U}#XZ>94?P#^(5|<+vZ~5Qehx8#^~gEWSDjm| zrf@VLx0mOS@Qm)=yPc7;+KZfl~((~k4zZsWsqMAnjNuzc^^1#sD#j(1BYtt7_ zSc0v2W|>){=Dvu4ndw8%#3;{8~C&^%&<&){QP)$v%>`nkraN)%U4e z0k7waD%%g-$wSCbN&}P*ovd-pQR9cSg9e?j{H$;83+W7ozRuaVW%MVUKWyk!Rz@lo z)H%Ju;#aI7U4D)vsVXvvxhTm_36O(4V#9zD1kWZ-@NKqKe^g!7xv^6p;coSC6_!10 zWl+MvvJe{_O8;s1ew$3vZWc5rK8i*_$aU6zF&{n0*s5}Erz0QTS7fz8#`U}n5zkM~ zCzg@~jV%wQHYY}F3)x088!rd0*l8HsaV?G$mITcy4}~>Pa}3&r2%?wkHNv*BM%Y|= zY5J17AeCj(3IszozK+Zg0Xyr3)}^oRw-b(?!4eckhscMbdgN6RRAKGb({ zH}_(-A4ZCbx40o$YReTFsoXW_+S7WX?c`-uw6Qwyf(dieKW~7*fI^RQJu*NG zz-Vs_pLK#0qG?mL-dqXDmasPvP+F4Xf`HDcplj9LBm3wo5v}(zj7r#b9m^2CjRUlk1WgI7|ip zMtEfZAZqy92u-bE6~D6)WKZg6>K+aBUiLR1ck8-2FT0Og^bcNh_!{aREqn5dINz!_ zB_1Z%-hgwPnw7uWUIJ~o#K`sh`27CXYM6K~@y1J`SV@|*2K4in&ck8vC3mQR%YjVB zG{xEmiMSU121ooUmW8zkt;P@ZZr!&e?DG-&nH!>{>hL0<1NyenY0h_bKlm;10y4gP z>+UfwQk$A8lSunOl_@!`g1{te=rt{_%)9smEtfz%faQxNYrvUJb`nr_$&yA}GYD88 zM0L{MGpUCS!mPSR>n%-WOP~2_LgwV>y+6SyFid(kM65O|hXyIT(IV#$+1(uFx4)qM zgDU_*QYiV8a($@~%w;j8EOm39J%gNtu1{53nR6=cg@B%-IE-2fs2Qob=-d&ZHhXJK zEE8J9)Dp}VEd!|fydD~*3}(UW?B-$Q83d^*1NAWynxWU0zm;0lG z$ARM6v6HX#V>xh4gc^73Tk3{u`gmsEu5W`sJFo?*=%RUI_Svq|C4V0vT>kp?49rLZD6*|a1UUXJbYf>Ly1suRZ#(C(5 z8qu3bv)(en>)E`9f3&{dc+?8kE~mp}HEf;?H6lnM_&9a&^IOKT!M*$(%6P8(!VeQ{oZi>o?@R`n1IQ9a-sw5OG)|03ql^(iRMJ|+C(JO$ zO$F`NWHI37AsI6sJ18c;C3F!Ht6$D=^XJyE>vj1M7HY>!r5#~6u;l5TL#y>bJVRD@ z|D=R->M^;M`*V~x{cuVoh@^9CDwy+hy;D@W~w#F}q_z zLqEVb!L}vb-^9!%_};U>`6e>fjo7&UKC;KSKGWR!UZVc%oW}iyUDzTyHV)T)0v6s; zuoWW4>M3JpN1{;j7r`&+gK?^ZP5wLST!St9^3<4gyjx1VF&nQ~OO;bLs3lBNBlI#5 z66nI9(LB64d?~Kro{re0sC;mGTR%2_Hj^Y@v&52fs4I3vNCt}T@h0mEpywrj7H(v_}me3fPZ6cGzxnc-#g7k%%NoWLzHuh(jBH+&iUTs zr#;T)zw^vvi?Y-~&{jpz)GoajxAtgQ6tUq+wl4Q%X<=&bfd z&)!gnm#=?; zLj1pk_BZN*9iRl60pZ6>#t)E)^a2#{5&Z>VzcqFCj|QLLFuF|MgP+G57pp zZ}}2|zyOS&0ppA;z$-f-Szu&i`vpY*e*Ek&m+Joj(cdZrjM~#NykuIxC_aF(=U*I6 zX9H%we`D$WBRuadZ9a&qIIB9D|9Xj-06YIbLT>&XO$YL?7y&y^KzIZs(*JcPj}2h` zego2f(Xzb6Z!>fL#<9T2!2)Q&fVp7cl^u{vy*R=$0BelpB{=IBvV~q|ihl>lf2NIJ zLIM6w0K&@2`WyW5pE@_@7q3-7JjMJnj1EMKFuxoO41e7P6p-5e4yJ!VATzQt(f`dF z+2S|Z<3A~Wz@YqP;k@XO7#RtHIeS39^Vcn8*%r;kKcJ#%C@cJVubGl60BB|XWi^=wpedE5P zY3Qf9IC0hZG%0USXV4flBguCo9@&ME4Voa~OP-igaTAupqUD*Nz`g1Dc(|W^C75$a zYJ0v<1dkD?pg~0%3X7$D5iR_L$;3L>fcu!xQdabXh#!ZUG)~pk%Udgc^1_m)KvH{q z$Y@I=;JLaa|F&yIrfT{%E9}MNcJPo%#c(X6j@# zyy8?M&QF+GNP@9QwgPkM(kNO>jWmrmFWI5QN- zl0+~->N|+UV#P^l&)%XN`zWq3k(s|Yn_ruLIMguFSM09IT$AcoAune?p|X~iYEUzp zY#!O$e@)5;Yeg@#(4RwP;(K-e+Q?d_a*g1qT%rta7H(2ihIf)ZR$qw&nHZ&!{JQM4$aJTLWic3nc ze!L;+z09X^)o6mfd+M%!=CO#6l1%>qC2Kz}#+E{p`FJJEl=5z2Gvo_H%LLY}1E<2c zkVJ!4gmvcQTzl>piNg6XUyH{&&F$jvp`iXQ7hZkhRG|^50$(an4)2}S1>m2(f2)267A3thtBWJChXLwQxUMGd3f>@^Nd)}jagnwbg zC5qr-nylguQe@NapvxL}?ZAfkT!+JB$y99{UX)o4CD`Sw;r|7aft*%b%aDV1sOb99sFns~ik`3VH< zZ{w-A)w%o|9zT?y8?_?dxWe3chh4k2N4~zPx^?r^;paanD#M)qO5pW$Hq_moxl>#K zkI~cymk~GPW^ly&3RFiqaIM7&J*7owLu4re^$|h(-O57bwNTBe5Xr5(=^f+40q%~xX>;s7yxnp?45Zs{r!yZW<~9hhm8XDsHrE=0gq4R?rkh|gMPwm-yIT_%aCxwkoq8!j=Cix2OuAy+{A+`of&Y-%FM z;&;h`1@Vcx{mkDh0N&MC6Q8_LQDiFMuYb^#KhkbSJdI)tKd&Mvc0O z(`r%~r(O~=xvo8hZAh!oot0;7DPjt=z@*a&ecc~uZc*z%riAG6X#7fdKAc^BRGrFo zjn2X^t~LhVx!r#@{5&78``IH};B+Jow`)&X*Df zdxh1wt`^kZT8;u<_o8lTDkWR%Oh5d#0e?)k0NL{*V#svS z?77l3v)Ob-Pw#8Zv9~uAW=`-@u|N||6`#5ujRQSJcU~-FaB7hg^K3!h@`7nVo?pN& ztkg_IynDo<3gUaww!T0+c<7)Wq0Iwi6LMTGzT5#G!p%2Am@6O*STyhNyMyUGydzzk z`<_%gMs1yu-MuSq3tet%t+T3z>Kudp)KQ7Mky6X5#`dEI3oK1fl1rax@V}e0a*rMD zIaJ*F2ol<3$|7@KA2fEq)uYu+=~gQ*N^2O8og^@%fgDRUVU zC3$yC_e(AsNWYVkA2q5H*e>f%z`2M(x}djQOa3bmuoV;S;VlE0)ecghLDPuUwxF-% zuF0igMMrk%U=g)}1)?zfV1)+Xlzgl3`i0-YF^m{(W0rsM6m{YvsR#*lX)XAmY0%noCE z3!sh_VJGROo>kOuG@(Y}jh_`X-*PP^Id2rTsm zF(+gwDH|j)4`g5|3&{eerKM8?hv62omTbP^AiC?vma$Lcu=#9xT)2UUAC!}M(39H> z*)u0Ro{_2@*zyN5m(2}`9Ft3zcG1YhRubCk9pv6R!<_{$?ippZ3?12r+~~dv3OW|* zKE%D@HC$3ExgZ07@{3bIoW1TP%H-GmuE?r+W>#6Ia_J0>5sAZ)aCWm61IcT`7}ggl zjgdj%WKYj2qqP*Q&A6J#NR$Ox(Yk{{6B(s1h*~`NYDf7?#NZJ_$Mh90(mNKo4wH}# z6k%q3*5{DUP~2nWz-2-Tk1IWdX@_4)PZzB z;}G$6xmz1>*xJf`joS1{o`3ZuqHgUL?Z_Eo9W+3vJ1hC77tFh8y$hV($V~NapGW6< zOY*g)eTB*Y$J|>7)wN~&!ng%@cL)~T-CY9&2p)n4cXxsZ4ekUB?i$?P-Q9w_ev7{6 zeBD?2)=9cwz4sTJT~M{h8gtII_V}4!CkHC4gje>@{b9{2{vX<>bZS18xTN)L)Rkv= z7HUoj1BRWBTj-3}jAL9=t9#j^@ieO2&XPWhH_f{Y`wdCx(9sWCb_v`pCNO$r<|+G1 zMQ=F?6>W>Pg$r3A;9ZD9Et%$VHZwtH{d4&Hz_z8v&?+NTVO_1Rbl(l!58dfSt5c=Y zu$epsd@dfzR`)jM#h@eRD3EhPR@1Ws9Gg^$zNof3;lQ{=7y~o_>B?;Rs9aRga|wY) zs+#xaaUPavyot#^Feq=Qouuaekf@xzT783@j$@9;jT|6sB)db1q<;XQ_iHiw!ghhQN zW@(D8!gM6o4tw15ieZgvM>GThjrVp=9@gj$BKk+t_^H(jE@|*~>+1JwC9foK(!+CL zufJx;A935o+6VeMpeJ7CwA3*MK`u@5@^!IkeB5$9vK^qG*qVmTC(hRE1H0h+`Yz3@ zG8PNkjW{U;M{uH(z!Mk2WHH;c4MJG#XbFg;@#5(@0xt&K;~1WL!7gbOtwf4xu$nzFq@IV4&v6;;JJuXb zoYl<1x)dO;lko^ z06e8D#3!ubu;-z`E514muFzT3cU3U5ybqbe?s@CLC1_o;$n2i7Cgib>O7rCQtgORy{EUs{)ZXE4?me=FhwXa$Z_GgU>&J#d5nJt*)mZxKQ480! z;YCd6XIiTby;-r%?Q+5ieMADGdMJpB&qDY4Gr4p6Wi>BG#*Vrw10=z2m3QL!hx>8P z^Z->U#yS|3vPp%D+z$Z@!FgPrH*bRVy_}GLc)l79&v1-flx_6GvSQ=X3rWQhd#SgxgBZj{(C6|g`3+pf%W3vKMH@*U*rGJz{ zM7ojw^o&G{dyr}K391j*#J&QAX#5@n#QPk&OlZdr0zK?8YItU#i^ES`R$Xm# z$w%C`;tea@S6s!jJyYGPFL?{_lQ*l)Qt z^lmYpW`$VORuPvWa}cQsT1DwYNY^$Zz)j2sg;9nLC`YT}JR}#z;t{&H z$RKm%&X)34=y$z=c)+`X_!f^~3e69>*QZkeZ*T+o4*A)EbHoFhf$E{H+(O>|2ZW>v zNE{iVxMG;biFn!iWCQ*Qqm}<+>^n`D>_rq7DEsBkUTZ4J8e+fjb!0p74{g1+69i)E z0>XBrDA0Tw*7A&(cx;a-ig5K(^&F4TUnCg^I3#UbrJSdU+DO2ZNH{>D>d9OLvTW1$ zG`L0W*94TqR*`)QB9m3CXzyTU1Bg1-sGlZ!h!rDT&;wz#0v7Rm7YRE0tP+TEm9c3A zJdkNb!BOUkPGMu<#srNQt~}bmwKXy^iCRB2zZ16!3L=e4_C**+dC)`08FpY)wVYM7 zp1i^Ope!wl#6Az3MirA$WwuZco0hOBOg4(?FAhL_mml92n1tOhQS%JSAmS#0}Uv#P;z9j+a}y@ono62g38Mn%Mt87e^0#@{$Xb}`EC zqWUHj^;0qhQ(vL`TZk2gE;9!{Cv_(kgsm?2T?(L z{K@Kf3rF^Pbfu$~x!9<+$sL~cd5F>q1lPFPk|5brNSz=q9z{+2Rm^pKODo(fx#-aE z-yhvKL_E_#PuSS@1s4jGpdCwEG5h9N8?8RsUst?Be^v`ev{exN_93abBc49ADW!PF zBWJG;i5$W;ma0tBUeA=iYwl2C`-M8x{#7EwQqod^rirxX_Z5p3n)L zI;Fnb3?`UhNNN;((*1>~#KXkL9obg?%y-K5Z>+GE-Y}sy&sxx-2n8IEs9)2fV?R8D zB$D9CzdpJA z6Q6k*{sSJTEC8wj#6fcao}qvq^A`qwzlG2Iw*f5R?#sl={u^bmym++&;02J!NY4g% zApbdl1(Nyb0e$OVA6))>K$+jJg2qw^0;+L4$mxM{cee$Kczh1^~K{Eej&dL^LNE>FY|F`U@p$e0SuD=0fK(X2V(?&|LcYQJ8_wRLt%gPixp7gzW6D; zi2Pn$Xn`_We*u;OkgDG}3;s#F0340M9LROkw>QzZWM=u@mi{^TR?gL28ccW_#5asD-*z*0nQ?3fYQMF;;Zrp z`vh=#2TZkp0saF5hJRzz{~xOAk4`ZGkpeFi0Ops#2Y^oUQY64e`GpntTaO0+?2u;X zU@$jjW;6L+;G7K@J^@A~fExrOV7~w2d;tKdzbGbWXa2{>g#Q($eX&^o&oC{JvdM|) zd0g7NK(j**cO2iGvuBYctB_|}%aghUJ@xK`g~b5b++IzKCunp9g;)*OR8udiq!2MI z7?I%k=H`=sN|USb)dTusm%(j2@B&v)8|6*!us*kX(umY=8_}ka-dnfhQ);A+I zx%v2vCn-xg=FXnrj}ql|py%_HrDDR#zHizQFAs<4RF4=c@5~6^K29F#48|!dJX-XC z9@9zD=V|=tf30xZw>n{_&AZu*iD|yALChE}doIQGM&7id>Ds@sg8OkKkmEHg1zpwX zsEV>@7MD!DUUPtx#t(5Nf9ZV}mo$%o@55~ZT#k(DSHJpMYSD@Nc+L_SyX21?wggVEhi73=kQ zZc5hRdCbD=76e2*Tb8%SNz4JcO?DM=F}&%%beG=n^SQ#a*f`E`KbkNyyV}cQlk$zc z>qO@|hCnQRFO!s(5hN}3cdKwI>M;twF-zORiijb}ATCzLYWK}lK9ukJ&Z@w2iQ5aU zk%@Pf>CqWliXUSU)KgP7WU;hwdLL?iQ)HHQEO|?75_^c7yRZKPOUv(WtmH6Wtpzct z#F=K0bL^`$PAIB2?&t>D65^u&ez51gM>);Ghf3!~5XN`He&t|>v6J{JYH?;{NS;mQ z?vvl<&lpFbiek4Ihs*~h0uGjXQjOSPM1Xm3w6xtUzdAUsP~!*Pp@E@`@=f%ro&=>V zW%<;dP(2b=X?OuDo&X!DFMMkQU1pU~2a8T}ddkW}T_x;ylqWKa37a)d4Zb{L(O!%= zCbT;Nk=+=N9Gq`U#Hd`8z0N6yN8+Z`Bx?H>RurhqzfhxlHvQ=2F}2w*WGKkfnRei! zZY_4*!nREl&6N#W`=_?WSkR)f+#a!Cl$N*-Qqj5Iyv$D3iW@;Knn6(wc zP@7sV*AVoJ4r-cW{x`Jsl7hzZY1O(yQJa`oP9saIwIxMeLYCAt)(dLs)zWwc{LL*k z-?tkx8jw6rpD@B|$@3kkNjf8(YSYo3=?rs@^X6QCd{5WW5+s|bXlMap%T(!oRl-Jw zdIPq?9Dkk87a#pv7R_jP{Sqq#G9&_fVXPg_)Gf%_10%MkOb z`5wtbmE4oiU75V5aO?wuo_IPg@eIwIQw>^B?-1@6YzH2hBTI~Lxj!vmk0Plz!xfcQ z?%#jUw@CmIY~~rMymxuBk07Oi)!~+nb<+4cv~&C5{@&0@2O*I$*p0}_;;dMGgtt6R zaG;s>!r?2%0BP^uj&CyDy76a1BgKC7gCHo~oIY1KjW(xf+~c(Mtq*Q}CvWxlo=-|z z2f%gSHpOPfQqERQNQ+1YjAiG-PnxOpZeLqa?)G~~o;}3UPNdw&>WT>!{ zFP8%c_sn})cN5>xf9BFUW0|0T_k2FNzUsEls_VMX64*H4es(#C*5oSqb)fa|Yfmkb zi^KO+Zd>Z;fj;u|9o6YDZtG5}Rm8+Lin&RX0bbX11~ac~#g9*v3y>HmU8;&?r8ayH zWO`=>XG7<0Q0tF9omr=~V8szUbu1peaWU>7E6U&^psNoz zwV-Fne4Ahh(7UvxoWu>UC3uM$S>D>`FhD6nL(U4-KO@LHMW;FEBJ+ZV`&wM&YM1lkZXSB70T;y+w`IGA)p09gg~be08&cUSJ- zg?Hv=->Cgg<7W0aLC00qzP@n`wv61$=q1jeD<%GFrz1Yz?iw z?c$+6`+PH(^BnVnB56bM*2yZV1CI!o=M_b|F(EDSBl{@-?fkXRnBl6s=I5%uTe&AP z&5vuv2g*sTY-fhfJvc7pkEqVeB?BL#>-kNII%N+}bjOJHf>< zyJ!!G0*r#DQY89L${5@FVa%>^1u!+o9nrya5DdbKTO&;~CfK#zREWqua_=w&kf|p~ z=zypo)Eqa)Q2#oi$VeC5epLN0!jk#WP*DXUt}ttFlmf#+bE(1>J?!`S8-{6D5Ja}a zq=QNm!%;wZl?Sk^HCmDP^4URTvmSz=MJ2`D!s}?_o$87Bt^L8-!$VZUKS5^Wwp*ce zkdi)#TPB5A(n1w8&01NA!wxewqD?*Oy`x${_jdj3_P>5v^-b z5ztdMI$CgM4F+0o)82U2GV(*D?*P9DOg^ANK$mMW{+}kkqIIxDP43uolvv`%~9h0gi2Zr2S{U^5Gas!-&C;Qb7Za>JZ{ip^P9yqsRuG! z*=Lry+&jiS_WrP!RN{6?ocg3YO>D}L03BE|Rx_i!D#KRiJhRXJY`0;=f5E<@p$YQuuNM0c zgERekT@6M1m6;dW_SxjHHztu@S`9??QpHX`kiQ-w;fVU&$euur*8GP?3!_;o7@ zP7(CgYmIysmQOS@>b~H@$N**BAypRzLE7wYdv^RN;LFJL1=$A^nJtt&sy3 z&2^tebX-gvFoa8#`a5tamrc<_H|l~nqEYg+_ZI33z}Z=aYvQ8d^h6jNA@T4(aQNX^ z3_uN)M6zz#gkWu0c_)A1tLECbBDdDdmTiGRqZwM+ERVzytji`Ww2AT7hQuJ7ssUc` zOflu;z!THsE^X*SRngr>E*bS_Zeg!Qe0pOmQneW8DKMLd97;K_{&wC{K%hsbkL{E<}m~4u&ds zc^r6)NbA#+`9dE}4bM&=spX!|-ukv4GU86enO9EO1sB4AF`&kyLgYSC&{Mz+KFc)0 z&_&j%6I;&-QlhoMw65LGL1TTSy0IAIxZ)I{=CIPWvvIfnsEbv~ibpodzQTz5NYQggLT49c z%aBg_{S)TlnMi^R$XbF+nTyH>&_V(U-^lnRCm_*dzZ6J9uUnHvF_9i>!+=nb@qWiZ zp@#q#&!_H4q)b$8PCN=Oy|?t{v70>`gqxomX}P13p!pg3+WNy#weO>0Fqxu|tkHy> z#Ecyo&&t~^ka5Tw10r!x9sTD&1(|&*CD8EKx0pM4OmOLk2CxXL{Jw>+h-^67s3^QwLt*YXW-p|gYU5GAC%313n0e3dBZy>|cBwq6hI~g3 ziuQR{im6)jVSUU5j6)9<_nmKz2?#tD@pK@mCb&w%Z2;YyB}{C3S=-bhET36AKG+4% z1(5f;SU5(IU)M+!f*nHS7mv-BZxLR-daf#|FbqE^a#3{)S@`IETysr>L2FSv z%*T+`hrUFm!+Kl9iT8Q&ehOXW%+tON9It6?DlRXG{BSMK5&-&%;}ml7?Ld|FoW?+y zG+(*Y(z=e-(RjnO!0lx53fQo+7xVg#)3cwWun9#&94}pH=x+t;YlAuq2Zn8Bdt`>Bbs34W$J)Ym*h^ z3r86(a35=9tT(RjWGWVduPI}_;7SAhUXQ$rC;GAAkCvViy}4nlBj5blV~IV{vXC`T zIYY#}Wdl~B8m8DHN1Ec`5bd;2mDq743M@0YdH8|Ma;Q%{1qM0cqoQ^>u1D|pXAGnx zUdgMRX1zLh@%GOiS~b;8jr!>}rk+p!LDkyTAYRbKj2|~vj+H~TuNFQJx`skszI=}? zL47Dx%90~C4{XtQkHN5vPTaJx5zu_*C~VIVcy1oV^nA{>uo{83+=_W)Lc-Ep6?*mGOP+Rqa6%k9%9ED9v5lmB8NQt<{JREqlb z4U~az6iN?=>%R9KBf&WJP_wSNr+V<%(A8S9U`rClT#cFz#QMN@m>v%MaTWJwjp0hw z{2MxCff4r1Bwf$B@?E-)U2z+Gbz3IOKFH4&>(D}NueC9uBtsda*Ej>^bLY6voepdp zON)#duu%#=W-hTPcAX1w%r&WdUGv;7POpo-^C(Vr-QVn>JA4=rHi=Gcl!Z%pt)Poh z=#JRCHYlJ|^1ZznR$7H%?c4OGu?lp=&k70D`xvpp`E<&L1cpVIjN>Cw%x+!OepM_pf3P@SX6;J0IVkq&#sImU(=E2v5;@u#oXd*g$^kR2^PfcGLrjbl@ky z5ZOG%PUp53qfks|XfPr2-Mvf>Hd=J|Fk@B#WQ>HK)9Plw1P+Ucp>ewPeXgD*#(L{Pd+&+4-IwO z8*N86rPn^`@i>-?c)ILe?n!EC<5YXiUu9?*S*&XACo}t=;EA4=ec4j#Jx@(QO&KbQ zjuNoO)1c=MT+yg%eQaao`dw4g!KY~+U_q$d_YOl!Pv}aUw+Bk+K48E(kx7C^i`-W{ ze~vy7bTFcBES;IV>YbTuez%6AkD6kQ70FpUEtv7x$W5*OVUSA^$}Tdr99CwPvS_&v z1LKwn5i}?qrOo&{LIr76& zAiB6SfL=%(0fb&oiJ*&=R5N%H;^*E&xj98i2oAd$6$D{fwaxjJa}8?!TF6JIZTi3` zaQ}9h>^q`oanZeeSmdZuAlX^Xv8lVv_{4M%~sNGO&w9ts|v zgNfAI3Da_IWoWkks!r2tvO%2OG~eQ&4-qMWY)R&Hqd@sGSV;MI%H zft~pufvNxd=rZ$fAxi&DFUMlXC1opSXC~|N={H9HEX;tRH3u;V6DvInFz^0zs{E3` z%LGh{|1tyvc)R|_RQdl%O@GXkUtn^!m$@r|G6Nxj9Kf8MmH8JyCE#y=qoV(2xcV~x zr#G~)G`D_Zp>Jvld}0|RmnKzj;I{n=lNxw?9_olIqQet4SWU^&VVg20A>J)6YGmP{NI`c{Cq*2Sbl-x0HB}0(X4;A{8$a0 zotaEjjf`x5V@Cn5}#*tH&(ORBeOu_Wmm)ncV^aW&kiFSR7J^%9n2RKK-f$=NL?H}Q$ zzl_=bvmNxWl=R1PV*zaZUr^^i(DWDI_?JxhzrKI|@uvTm3;M^C&eG7q*3{aL-oSvt zTHo5xR@cbX+|cenUjNM!A8=qdH)MFR(`Nt3#j4L{WMrVv`Oy#{%`kFu=zriaG6d3Y zjov7}{PP z|L%qPW4!>^1c*s^+0#Ix06_O)<^ZJgzYwAQV*LFZoBF@77T8~oeJ((s`|AyU@t1fZ zk}(5b`7FSL%m3eC;2!^Cpz+sN{_j1x{AV7?zfj1Z4FPWE%m82UkGScVvY3D6p3eGP zk1GGAA1|yPfK9;u8y*b*!A=5n-_+(!N0z+{)IyR=m!(~ zOF9MM-~S) zsY(4yQYiz$d^~fBT-9PSv0Y4JW}^L4+cQ=iAaAo|Wo8)}&|$oXpkVC1JbZflhG4l| z9)oyK?hp6J=4dI0U@2^bQC)4>1TvXO@Za~r@{N(ROQ)KU82A+r#0X?DobLGtlQ)=n zo%bk4lDi(zmSEmq)L61_88|(6&$KL~A6%4H4SGE0Dfi1xDoG9CtwhGLDD0`6*L1kN zXE-yQQn4N}ME%jblQ$`PE|;pdwYmP|;46dLr=pUhiuozo!U`MJuFVSpH7fqaa*ja-niyuapWAFu{shA10TWWv~K0( zf>PT{?v?I+yOUcS2FJk#z1tyoJ&MR9&&rd#(rvDJ9LH3HXIPVftIOr?vBV?`Lu?Q= z%IhQwCZ)TN4el2`UcG(rxHhr8Os{n}B=-C;XTt))2o0*eLF=WU-^ASIA5VOIvN)}%Q%fgQ6;`6iZCt~c^asz=Nh&q`IPFkbg}K+Bg8mz^0RyBLC6;Glg40!!$et zh0gTSPXZlV_x5GaE}6y40q(Q7$);>$So4!|ia1Xj6>DKZstWAsEB`a~kj|mmL)|9% zSGOsJF+p2M@`5ISt<3SqnQhvfL#hv* zqw3V+Bj(8i<&V?q0kb%aYFe+MWU;(sS=|+CY+_@bm7(VNJ?8YRnzb?W)ZyZOgQ$rl zjdz;iuba9JI8kEfm_(lQx;70Gx1Ybgn-e*3TREY7^QrXGxp29`_%#uby&78NapIk} zF<#=9!wLiGplD9cly_K`25#m%{uBSA6Utg?89tvU3Z~Ep%vsQ(2O1(!Ql%&nj>KnQ zo+2<1bP`o7yRpMt)kaqd4bqOZ$C@6t*CGwVhF+*?ydwlRpDFc%%L;SOFX_EhWL=+- zJ4ky31*NDmEv&APo6xJb%J5X&U}&I7kWE;fxv?Tih?Lsk-(ZPJ^li>NM#E4OoU-nl z1WH}su02tq4-(>Wz_FVaw{1l0)S>}ZX~@8EYA zmXFT(YKK2h#-?(~cNLhy#{$P`oXx3_K?FNrzt|~C^p0IpF62k*J&DC<1PH_fN}LYf z+7%6`VqC#zi8AW9%K2Q5i7h6eJsw9W5fRDmmV#m4#n`0m5qL0?Z%}eao;|?zv2I`) zZ(Ac5`PWIdws+E67o*@p~DQLzJ%ytOmU~@ zJRQvpC;uBhj_V3Ndr%yYu4W;TE(YDIl9r^4`*;bpHzoMgFa0CTk*mQ_d zcw;|7o}_wo0KK|$c<=-J|oW7)DZ@cT|3IuBJ?*4+@fX2&J6k`8Lnx%Ys--~k-@8f*j3{=hlkP%Hj6o5DElMEB8l4lo zOC(B_xE$T6bf_%YWqvch(3J~{Zyrxpj!9{Mv*hHghGCLma)cXery&%0UPM(URHeXs zsZa-NAHJka^q@@iDj{qsZ@DH+cETZwZD`|>Zypib6QyJxw2=Yipz+>X(KD{ts`3mo z7rO;*23i)rdz32jH7cuI`-LPLcmXxCH9@ElB@_$kXIPjuQ1d8~l;JR_bG{&9?|?S= zU5=9bZ0rg~b+G>Be3URdu+lC4Fd_~{Ju7h1c?0*+)>Rdakqww(YDphFwZ95Z?NMb__QENVbN!E}YI{CrI`q2p1Vl`8#5(z}6 zH)AevN~kTGx0h2o ze3Hx0P67&)ENk!gq}Ti%tA}mZ>;B>Ei{9<=Lg-&gwaB3i4o)9e+r`KtXxKixeA-qz zjlmRn8&elb^#1YQD9u}<=QP5G7>E6y_nkx&`1FzN$MokB4Fs}Q0!Hmcma|qfl>AC) zN9#|UYf$X4@g`eBpha6Aczq>q!6@o`RPL@`r*jhRsSdqW5usLax(K;L_C;+~@15MZ zbubT{tD%&y4iP(tG@|^vW;;WTG#*$Dq->2Srqaqq^Eu-3XFpg9I~ovAb_tf%u*78y z3pqGHx!0l?5HI4%pnYLN-i4#|=J=9vZ?x5L3@K7Zoho119=N0Z8a`sxD(d|~Up?*T zFc=W6$rJXIm9Dq;72!F8xAY3ynHqUq2}*S=lyc1%+Q~(hjJY{fNy^|i*qG&Q@lrQn zftNh_kT!!kokDX*WnTs!&9Ikzc`-;gfAGc`VJ}M`7{L-}3bvk;MR!BBUh*H@Hr(?2 zP&=G{zaJMTEy8Dl5`2wYEd~PXE#LEv;sZh3oq_ZFxMU%38KDQt>}Kzb4EpE{24r7r zpQKec-Bqe~$RFn91Z;3BxXdP`!o($?qvlPQaJq;)A^j2xl)|!k)FFQ^d7Gnr`@CP& zb0e}x`lg+NZrj$pMuPSjh8Tk|_Geq0mLZ81^cp+559p%*Dzo>!w+a%mk!Irx=)84sO0BaS7iGl7|YC?z>RN`5MrY!B<4r}FFmWLw??0yu^cw{8(1 ztAAJ?I+&KW%szeD9@Kd@ENxe|+ar-xKQ>_y*j(UfxN+crzVFdTQS#6tUsY2+7Ioy8 z0gv2mXc)+=^|F1V9j;)=svUn`lz_P`_fU}{mt+=Q?a}`|wWt&2p#o776OuBfZS7A4)y3EDZ0lA8Tors!uhXr=tLt2?Du4vB{bLUQrV_a zU_@T1?YRr&lIzOCCKt?(NFKwX{f35j-@afquia+J9s4KOHlH;mEgNky4~;z+W_gyt z{A|!prKJ1L;k9);CX2$r9N_)4ugBG9K|>hL|1-A@bu%uVF9;?lc7|X1r!c%kWH?ZE zHdJt?WxUQvDLElHybQL_R;@(h5D0v&!7@1^!BHN=(5ZFkjvummzEQoE>z-$$MlvtVcRsH2tEQJ3D$L;PF>K9Z%Ly}8v6e}ajx z5ixd!!7LX1oyL%@`x3qan~)jv+=MbxUJ}e~o%v*gP;y)naTJkWIx$<@Cb+sjtk;;RykY+)*Tib-#~4S2Ma7dtX!7XLWr*pBE~p??O<6%=xfhQw8q6aeA|rTVTbKTt$-o2yyK6X0^1Wl;xh_ zbZE$Hs)UI5>qtu@z204k)}Cu#H-87zW4@N2P8L)jELgyo%WPXb+_42F{zD1XvQcm6 zR=0XNUhKEZq|@GvB9h+11`^{Cv0b#DS4-T5#vb2^Ie%+)T}%nJm+?Q9+0Z)dpmb zl6RUL)`gl%mfilak5EUC1RYHH%he&+LFJ=$=h*zDO4!=+{A8;L2y7T;?}m7_pikMr z^N#t}#8t((Y2UpkpCA{mnEbt#llQ>grT^D{fz4?HD4~#DWt|jxR@WJH9oD-8)`kmo z*9%te#Y3^o#-UmxXUz|$OH0f8)W?=BA7GX7v|EbEtoDX@T;p1ytm4)0x|Ef6>n3(< zCnPn*pk%O|Nt{2;-K_GtK$PSBbUT|Ih`2?fvEGe7{A7(7?d!s@TsYDBdWCs(J-KHX zq1tY-?=DMmZt13G1?l=J@_rtxBcitdY%x78>l#tIetzY_K!>B*FUt=?^|F(KTvx?G z8)Ve}5%hlO^`(4Cfo`D6ae-5E^fV#(K89TsszztT98!x9`U%kDpLpxr2ZI+58w14o)h6YME1MJ>^;d#{nFVDdQ%OG!1anM)r8<27puep@ridv@LcAR~$>=y| z@T507Vt{6Zx@u)UI(#e}#~6Ci?(2vjLoI@hT(07WsMI=wE_(`sN}|M*0cFxC5Com& z=*+)dmI_*HEP+}n2d~1tKL2i?eg0w%kk6(m$tAHJ%xCc50mTiE6(MAI22?Xz)agB1VOT83QN3YVDunWvF2J z&Z$-@`puRB?&ou;uGHp#G2iX-%V^u1si5!Eyk#PkXq%f!nq8i|Ej-3el8En znl&6@z|Hv7O^8!7Se2$>;h=mC)?~}o)DdyefriDA>MbvWb3<_@j=rLYG}+480-^*1 zH*UOUMcU_O!UN|cyi$1~&>d48fyj{nuVdnGfuIcI4t|Lr+n+Vy+m&(apF{fwT)#Gg z6na1sZ!FK@U)0)r!yMBQw|kzWi+odu6zs)tr8bRoaC4Pe%Z=R*1o^=T0w?Hq4dqXw zjl;&Hb|4r`cNn_ZoDKs9rg+@dGBP}VS^F@|-7$V}7e>uX7S(!nZyJi|rx86-%o4`3 zOwYNdC9iJRd@2Ls|5ck5e*SE7#ylS1{xP=1>T$x7q^zW4GxT_(guYG{0@-OwcD#?x ze&lCfe6T7P_xGdmPyRVKvKn$VD2_L#TGfvSe&$Q{de_L50V|z1a*!|!T?0#rIg8I% z)5z}*efNjIet+YC)ss5!j!5rCmk!=aX2f$*ckqz$Xhpx)!EE&&lvU5W(lyi{nL?ag zviHX&F-QeX&nQOP@&40p-@G&j-M!|-B-r7*kvloo>k&Ncnvs%CTGo@Kx zpYc1bwW!LfeCeu)AR(d9zE>=Z88MSQDjsK@%MvnI{O_*?aNV9P{AnuWK%|WK`q3iE z&Au-bU3C3SH(7+{5%0!@Z8Kgw)0@x16|qm<%`A>X zMlJ;24TwTf83nHSYL7NyUDD@Q?;fvyYt^6djVydA zp07>|EFC_KTfAp9x#zGfa_hS9zjofIO=GB#i|?G^zal*dTjOZnuP8V(-M@F(-! zxI9Uall(oYm|c$df`O6PLO?z04U<$hopZQb^0GLP9Ub0m4)LsFisX0RzN7o0#F42`~v`y{~TKb z_y#=_;NSY!=e@s$2>v@u`D1VmXi@=S1Q^5vgKNNg_-~c5{fe-b^=+!G0mEPmeG4lfM*er@+kpQ$pxb^iLICIw07Ug?i`anH{gP!cpkx1y)%K52-oLl7 zKN|LeRj_k%5p%s5Jp-f;E&xCI1t=PTvwx$i|F-Y|<_*)|J*xqm+~3(u9DoSQGl-dLPl=Zmdj{7J6fzwW0bMdsWlOw65@kn2D?Q<*mm@DO6mq+xPS$=Xrcg zGlDU8GsH0pmF8Rs7A^iY@r|P=5+-$Yntc3U=;ZqMQ;mIYZs}GaKhX8nHZvvXWj}j3 zyMb-6>sNiw)fzw>(&&dg&wgaApqneU8W^G>5(t!_mKxsKqD)U29F#9n_%-V)#`L$&eY7ES=~#XohO~!U0b6FDWKMcS@caD+diezD@J^dy1`Xr$T6Y>OlaH5Ll(MqGOE`x4y z=+@Qg0kvOZmeTPKHJR)49XD(gDvAhbkMzwL3Ne#W5}CE`5f_?h4*HW)tEX(VKMG~H zC;h^wa)UMM!nf3)UMZMFX&DL1*lth5vAZX6#r06GNZehLbIExkBB^cyrB|q2!n}#wCr`htpED0>Togb&TeT8k8?2zHpu`gWz!ooo29)U&bqI*|h0%)|n01JGPdugEXzPvdW)3S5nkm+xW-ZxteS(Sv z*l`sFk&8&Q>F6g8>VRnM$DVE%f z*m2QRFr;kn?Taz9_1Z$QQN82xH(n7In9?L=QwcbMKwrHMK@u8<@o~zh+GJLZB^3KI zsVB!CWo3M8unP|vI_8(5{Hh_iI}(sNU)4DqhkXlUi((PBfVKz4xBN-G3r1K?7jj@I zUC)U~kPo}u?x{mBu%m;T3Gc!Rhp24mkIM`vPhPY{HLYL3ezr>-nP5_^73mORqns*D z0=))xY2va7v9FR(xZ0|vvOsyJ7OGlK-%Z?4gd~_a+3iu|2R@1~s=v6tHoC#NL&iq! zU-bKd+dn6R*lsuDm_e@q_Jcu=VT6s(WUPtDbC#bSM1oITrBH9ZBv z4=qg+wu1NLXEX~()7tK2tk!P`zUBJAlMm>Q6f~&qA)YpIukIX%oJDCgPa>)c{`#@t zgTx!@T_OK3<-r(|5;F6ygmetnh=iYHhR42zgGTOKvW=V;ibjDb&x8=NAx)CwB3ue( zBwk|ILtOix2#sRcH7mkjc=AO|j$)Y!9#5XP8fCm2BHqmajw?-9RPuZ^<Kc)op*n3^?E6}J-kOwhx7Y7%I)2EA} z1-lz0&k4uXXx{)|?tE875mam|R7#Du+LhHDc?gz6hVG1bLlWG}h6^R})Q7hXH!$pJ zBO!*{gtdLDE*x1ZdQ}hwnH|{lBkakxT89Q{xp1G%BC$iKER$2;Q5Bnu$tT#DFT=nv zNyWUnX>a^c`KB8$tc@y7w!ghg%T%~;H0(aCOhF<~m_o&x!1dWlH0B@_ zS+fJH1o*BvBhb+k2hqW#XN6WuUC1`rHXy=Z)RAOSTR3!XJW_Frzmoi>O*0{W;=W2t zq0p(aQvpJ6rvNl`J=QlH6s~;;uhn%L=A19-r$EduX9k3-kV%3D*2Fk>{U9d&(Q7KH z<$#)NAr%FY`+?CXnfl?H%%99vKYF!!t#}9Gk{S3Puw1nQ2#Lhsl6QeozB~VRS@mky zI9aZ_IrUiTo)~xW2jUOQ?#j|R*ZVicStj(@ng&n>LJ$Rv74g>t=1O6Fc-6l9l!@_M z@#b;HXkG$%5$9Y`P<&p=fzXVKL41_Abt(Rw4Q(JXpHhvF(B#eqt5Fq?wzE<;&xZ)# zSd`QEx}GS3MKsye#om+N96CI`Nd`+sU#XDdciXpWCQ{0c8QtJ>FV%#nZqVc~y&>vO z*-`L_Um?7sT~wTla|&p&Q~kKs7NMGt>6J2K+hUu{VSk1?uP+4A(DM1nbEexvz$)0u zpS^8Qipx3`Z|hZxRNImpqssS-a+BA0PZ^4O7~X9Bth2Ch!`rtA`a>?Tfe75qWFEAlJ}9)k;kZ>pno{Dv=*M1^%^|F91bV&Cd5osWtAVj z5DSLLhi9Zs-sQ;_1yk#bq@bnqp}-UI2~~+Cv;qx+?MSY(dgsEQMsR^jo{;o*ln4zk zwsi)|{n-l@FAT&RZ@Y9Eu5GU*)!Z5;;F+SX*bZes4TZf3+Bm zqNI$XpGnM=lkB&9=a>Ti=`9ZoM-eRnG`^OGnQFu~s9R5!^DZ-<-~BvB-Z(MeCgaU& zp`ywJ%88i&Fz85NIpsqruOnOy~DL(z)S;+L{C&h9T zA3~9O10S`yC*Z%9nk#2l z9F7qT>@pGHDzspT<{~{qu+E7vr;(xJyx$@1$z{~W43OHHdWhC-Qkc7PFfd~k=3BbM zaa?Kmj=tm5QdMoRe^d?-B7^;EELK>+)?RSK*~ zZouAR78^1FgXCi%PROcfJ-nFp4}|v^8(`TdB$$rx#Q25HMc52*zoSe<^5?EuO`^V1 z?JaAkO)pH?$*Y~COuxz7AB#QuN@X1*8wjby>If@eU*ap&Pw8~qVBPHfYP&U~>Kitn zOHrCqmS6QNTkf2b2A_SLPW)}CL!0g{Bx^PC9zo=S!z=iX?)MPE2_U{~t4U?sRNR*= zptXt*j%hOYJ9R)cCU8JK%8zZx2#IvYG%|Xc6DBiR&5DYd;ooNR2Q|y7z z&qpP7m#Pu<4p~;y%xKRYCLqe(5Z~xYb}41F5AB?yqbB=0jDtiZyG5u{=hWy+*?M4Al`ZhK-8aoVd;>Mcz6R!XBsu zdBF}1*8Xu6gcH1ETM0ZUXpe-gP!oJNY|qCfxk9M5O(;sXD-xMebS$1n_-*(XDT13u zRl`pLpRIgyb3~v(_E~aNBoMrFzDoptBQANtdxvqa4lG0*;|u~#tg2tYMqY-q*E}JL!xydEu##*6U1#- zH9$B4J`;Jzt{R_yP-Qf;a*I9@od4a=k4jBD2#9=ZZIBL{cVTru%)A((Jhcp6%I$kh z>KwjsvZuvU(0z_X@aAZW=n;MolHGS;&XLu(9V$S>nr6ZDP0!`*$T1m%zGADvYiP&L zi*0zj>&?+Rv@q@Q?ut=uSJNyeHhTNEHpYzQ?Zl0?HO6`YHb=t3A^!?G&e0Hf%EE_W z28S>8yBAyqB$kghuVe*pNBKTwd_J&3q8{Ga)X5dsYemmFLc@BJ)Pe}c&O^&zlQx0R zX33yGC1Ho*!XAWIXNGd=geAzJ3^pS@=FiNAM^^L8_@siRgbuibdlY2LO&S{;TQ#hs zv9k%ffS5<#oOESVA{1dPGVyqp{eR58WmKKpmMw}e+#$FHcXxLP1PczqU4sXAcL>2D zNbm%AcbDK2TtaYncpv+mUHjboPTjYYSJiI)Vy)F^v^C}!b9|H8`{*5dn5qwgV}E10 zR0*6_1{p8t4Gou41SPE%rl*3bHbL9I;4*}=E2_gs2Lg^D2o#%(Kuw*&3JbEa8Es+* zx<;<@FupYoeZ)fH!ti)5|3B zUz=T8d?eeFY=l%kWRVmu?fxnMg?re0JPl>SpeVuBb-vroTxM@{+HgaGXSj=AE5XE2{~4OvOeSPSnpa)1lYHf%z69SQ@D-#zXqGuficKj)+-2 zUO%Ib?*o&NlW&HfEVv^>uy{8qTVAjxBGP?N<>DJ92UPp&H`UQ(;A)|V9B(kS?3|Z| zn)zM_Mz<=uqq;eb*i2pLzr*`7HUY|QeV7*dxED0$aj+CP>xsId-|D!j-foP!@Pg(e zV;Z3Gp{l?M&lNxkUhQ3+5=I6-6n~Y62F6h$=LnTHXyU>eSk)2}baPbE2w`Nyx^Mdy zYVFC28pd)2VRPSDqt3C;o+I{tHI2S2k1_&Yh6+y5>djWp0(9%+e0~;=p(D zGLKJ*LUrC~<=QuGk>9*WJ{@{x9@Z2%?-0eZ54S{QS=+3ZR}`-K`D?oB zdU|KK2~`)@LU(Ak1(Uj4`O)1an{Qgq^dPO{V{(v($q!0O<*Lfym!qG=({%cA5cS8X zy-8RjBIy0aBgNpR?Bh6sCw3M6S%Rh7LP!d-=X33)>Ju1M8{wK3)1Wl-xCsOCp<=^~ z(nA^KdO5gp8G7R~w3Ev(zn>J5&aqTP_aYHX7Tsa&h33icn+JFN7*88E%ZaVehAE(2b&1EG^GA(e~zKlWL+%kpDV3Tg!Xg zP{OjaICMH(kPVMiEZd6SRzJ79S@y#&T#h~`Uj+l8=uqLY-{p9`WB`Uh z(5ybB-V3AO(}LNyc4d!ONtcDsWotaOr^_qsS^*jsjUYi&-3MP%%1;yHR@@iiL*s?w z#rDXigm>4e09jiK3u)|w$Gk@Li&iPKKn#PWIHL9Vv`{8iP^~hMD`O7h<#OCBJSpjQ zS?GYmi>P?5w4jWqyk-m@&7g}^hqD&#-hzi;h@RcyVMm^T2Zns)b^71K0 zSFZX&G1YN3>!G+hjjRQ!gQh-5Z+asseu(AFa@tGXxF>L&h${fg0}J*l2Zq@B=P=Ye zWiju~sT4UD2`?~DdI||mOi*t+AEc#DP0h@NJ4#%YH>ywJ6{;9n>#`W6k8ukq7%QTE zQ_y=q;VLifjV196P3j^D@|nS%Z)nkc8K6NY_P3w z&&q#FNadC8FTDGG5jeA+_*$YxhYzH<(I#bLIC#yWx9^i#*}wDK!r%EL63x)dDBy<}tzzNi2HBdm z8dQCkKYSR{*+ObUf$FOJ5eaSd!bdHi4>JOKNJF5081bD%pkKC-vp*7wbKpsP;`(=x ze6oV}C+sCDx?m!9zunLN2jIOy<{YLv;W)6(d^bd|aB#+%fqx0esx>~(_xFUloLn1V1|`EXC{9YU+q?F8u9D`5fbEnIK9soAbf>d?N`{z(VwB%O!zj(QI6&P@ zc9&VFKBptj3qJAXJr$eKM6_1ItXqj8q@({v4bRe{nH@|q_Hz!U*A#mC3w`%hfE0ep zyN7_8LOS{q=Oerx*>aevZ(vL5`AH?j)sigZq0VF;v(%rv*7aC%z;u+=Z#g6@uR=`X z@R&ssCSxQ_PDoJZwpU{GIu|9TM5AB_$XG`3fAB$8b`*7Ns)ECtXfW4&n}MlA%MmSl z18=|QG<=8h$OA&8%~AVFg@zU_VX4tjD;S)rgclEfDK*nUi-mn)%3MYqUm;UX1F{A8 z^ifg3BpKjn9wS`*khk#3&U$@vvr)%*=-YHF+W}k0w+AJ3=15DdiR=+|@qq1O6>dnx zJiDE(>sS3*v^qDLV@*+CTY}~H_8p}Qi;)rRzI8>Cd%m%rBPK7xk@2maCnno{^dL`M zjfRMNeJLGFb{mcl`QVcxA92$C^no-d;l;AlVCz20e(oS51!CRm z@luwp*<*RPEidHF&m)`Jd2*%)h|8D*1jo^-(kC9BeB_8%0V*p`%7>S+O)f%- zY^}F6tF}2m`HDiRmYoq?E0(u!P8Gvp;&H(A{3zgg%d$j=myu8ts&D2!YHiDhV{zFb zkPW-Uqex7TAUCmWqOx80x>hy2M8ueJsMc9Zi7QV0X2RB@NSzKnH0aaJL)vS5GZ>Bv zP*tw_wOp&4k?*AEUv{+XO+vPB6bXJZ>W=CayA~kALG5CTbUvPauegt|(G>l}Ws^eV zcT+(rrfcq!DF9N6TgYvQCHFyt=^ZjCbFkg#7` zJFA*1H>Z}P+Hc@Y&xW-Cr1XrWb3IQF14g#oz=$^|pz+TEEdAd`137*RSvRwAGIus) zG_tc{GO)L^b8vCCF|ajb0?=F=dnX2KJ3A|9dj?|@BRgXY;NxEp?@#$>P8KdeC;2a* zRL*C#`WNQJ0L$3lNZ5aObboRQfbsn2Nl#Xw8d(4%;pbq10DmEOzXiGfdvMywngQTy z@cc%up1pb50EC?bkQxT&`hW>`V0IKp87H7X{tH$FfZO+Pr0PH0TaMNSTx#6R&gvq+ zUVP8u0$hJ+jsr>%Y=3z~Z~%_pT)*O?%<)@D`+rABf07{p)(R(JbHN7q^>RP&s()}J z{&utA{4JdQe;b4S!*-aNh50weVE?r3W`B0^07R*uZ4{pE6xjcuM*M=20x+}w8^!kT zb`A%lsfF!xmvdyWH2{8rk&TV}H#X68B+ujyHty#^<-ce+Y&^hZEI>l}`ziX@Q^4PT zMEy5L_RrE|fa|}a+5z?`z~}nwLCW%MO7IKvDZruPH}d;GY%%^>>?{D+ zjT>m_e-=9nz;)yP1@<22Zyl5Vjxz^ zc22;>=nr}u2XNH;vzh%v)Ztf}*>9bb{(EBjXPW-W-(%qhvcvt${5`IJJtzI2_cElc3kAHe3qwpsoNJE#ur+|ubCW80$> zWHvc#b!hTDl?vG0esBKW9zr^ieDt^6*|Bd@k8eHQTrR_99K}@(RbIsiE)jPm)QqPQ zO}xxN&z*|Hj>p;0Abv>FiQiD6w0207uQG|txYlbU(oPD@9I{X|tddJuJXjrjuNgBk zSm9HT+`=_|nw*-q;K`63^a_$;%c55E?$UGpsZ^~;>~JO7EZ0IQn0U`b?ThJrW1;NI zagl~X%+uN00U8C3stC4N*ynp`N6C8qx7cnpOtaVVa+R7aG&{$RTW+fZ3el;ujG@F} zWS?ccT|Y9#a45X?+yQwaLo$|F;HEx#$yxkdR6V5>g3CJgZ#tZs`KpVRcC04ID7-lh zclx-$6=LQikn8sz93iis=i6}no88=r`+2)3nV8=38*H^mZO8Kgqzq!2SKKDY+q>EL8+t_`^6^zWyxrBF&FGQ~Hym@xJb7~#9F#ZF2`Ueo@3jD{=W&(gWNLpmCl z8rTQ3M{lxQ*W8tefMf-EMq0#EuTAkuhv44U%f5(o!Qp9ssJMBoMU7gDFhQz;EO8eH z8A>MQAen#A;L#ZqZZMSdG#gcVMEQFn#QAhx`YYK|g8=RjnGgKYfs~>b_Q+G!rt+hD zqc3_YaZIZ0p!MtJh4+wAee>Zjtq9;=7Ya7p#o;Nas`&1C3Y7H{YDL50k3Cj=8JTmdRAgd(e2wJayAYT%XTMf>;El&}#46mVlq|~-c(!C4rYOOF& z%p4UgWsRec4kS${UgnaPr<*i>qNSrsv;OwUp&paj)dBTM8Dl6{y8W|s2M+P39v(M- z<-pU-AY?Kv6`>kcjkrGxE=f>!dS1L_F)_h=oDyGWv63pLo}(f;y|*hmEdd~{mjY@& z#Hg`K9(E+(AOm}fFh7S=g~xL@Vh&Hy!*G zQOLF!BQQNsJ+yqzH~ccWZ|Nn$6j8&96QahhEWpPlQuK5 zWS!@9z)wLmD*k@oFO%oC;YB^1)rWj>&E1BKk5xUpCp093@6{5sF$pHvZ*CqwXuo^f zc0iYI_CcDAjzD&ymsjCBXv<*zoUrTOq9Gu}HzZkA3KhgtLa8TRlMW^M^Jx5*&&;p+ zim_F5XhlEs5;Wy=rOs4O0RQNKt$(YXt}@adLEXm|xH8;fXHzHeagst#uO^w(y>6A@ zv3VeAFT}8NaA>G2PC8NCunl6dB)iK}QuCoo; zsg|@4de+ujrdL+0`86DSJ@MI^PID>RZIH|Cl-3xY{c3wgshev%@N;w72^%^j<~TdK z@NvAA7WD3PgtXJB2)C#%C}@|=j{$L7=Rd^ukb9Fn3PvNo-D#CEIf7xAd!p%Wi{sAZ zBSP2n%XP1O4IyJ2JbU?(9+LvhWJ{zLm>EBu7aX^ZmL*Q0ZzbT}=Q-5De&u2r;E8B^@ zz=<9i6TQGeQcy17ofgO)^q4MqXI0@RSq_aUneEl*=fSJ$W!h=!58XpT;;bvNH8-!ZO z7qi35C4NU>kWE>7UG=kD$dF`Y(f;~`_Ua52V<$E#wQ84Ylh%=BC#82J>7mWkb68_h z^Uh<8yzWL1|IJfD)jes}&EW<5ab(@mF&zn4N+KyO+{!+z+zfX-S5(VXx;-H|z2nY3 z_jp}|L1#=04&|=>0d=}ASRuM;l)ovzJ^Pwf_@!GSraqS$-)J+#l2rbRgZ^rN0Mm;O za00gltSG}IGzw=#xN#3J7P&mrQd{_nklZ#+WJclq&ev9@Qobjmxl*L#X>PXk5_RpY zxV(CsxrEIHwzHwwf(3mA>`J=zIyr3Uu|$0Ms#WVT`!VcTe&Z;;n~9O8kA?m%HA2Wy zuZ}OF>AznlLR6M2(3p|k;G}-38n-9J6AZ2jvpGaHd@0!`k1n~I5-bH5az^%~e8a-o z(CU7(gaQ_O8S8U7R<*F*sCTHd-o^XWe5sJoJp8dqWt082l}NHLF7w+&#&K(K6va78 z=9KWW04XHu@GdpfP!)<89<<6bjZ_Ajv=vxzS)Ij$w$kgWWG ztl4>_JVMT9L4qp6X>Wqj9?pdYYfdT+aV>+J&KbBK{WY0e6cQ3H*c&9p z5AN{7Sj;r;dPs+0#UQ{fLwpxZQi6FqCf5T@_Vz`Bt+&9F0XDK`HA%CP&Bsh=NH4w% z(w99fnV6zpwWA9VnAhs&?^U?&+1T$}PQfr!8AVlWP$~{=%st|;%90+G>OLJf6XAUk z?J^npXnyJe{iB-Fjc~zRpq72$=UT^&4q7$@)a+e~w{DxjJEHK{Z}qK*u7+1} z(yyfe6`XXG?(8O=8ib@aWhRtFwS!U>VZ4$usg^N~LZT9LEyiBkQT z3{o=_k!_7Z=jiaLIdDw}yUbQM4Q}?=v`yL>9mcY9wVw@r5vCU2`=iTD36Vwq8?T+i)nrGsf5UEMo% zc)P(euB|V-hY`d9j1#PBQ1Y{A1DJ?TdCJZl68%x_?yoc+SsGQYlNn}vyf_^`U@60@ zYx*6icR#g2i3binU6gy}#I75jvJZy88sN60Ts*xI3!%=ZY`*<9> zCD;mU&8pP`PV1c&RV>bA?SQMH??gvE{eHIr?kmH{jWDJwPq52@o3P z3D}Yfr#|D5D*Hm{FufWzb4_dGY=}Yl2}Pt(dqTBXnM{-V;9G8cs$gzz;gI3OC@Hut zlW+#D!N2B5>6{+ir}jE!CQjzo=5>n8W};7HOZ|zL+MFq!!B zU_x##d)XhYsq!frP-@2tGKT2!N6OpDJa1@3Y_6eR*(CUhRWOZ-!P>Hk1=`$TaClhv zz10hh?Bzoxi14imwIF={i!d28|RS|@#g)-#TZy- zM9t7q>ZYMu)Z_h|QqfCS@P2zv#}iApis0_YS z8%xOO7Sv%TpGrScy?D<^&?Hu!o}j|C>HLo<1#a-w=-Afbewdwh~$4kg0}#jf?O zX<9>H?YKLJ7Soxki@TEXmbn%?g2nsOR&O?K|B(9S4BmNkxLVLpII3?OIB~3VPI!`C z-o~W`-l6lE?YO83PFEHn{9Y6isn(z2x47|mXlFKvirZWGZyZCWg_{rc1EjU-kwr{D?cPpP8SqGsA0kk@3x zE18*aH@P8}wJ9O`uoEkJ4t12~VlRQ3bI*c()wHJ+X1eW%MreABoLMGyLvI+(C zK6tae%6HbY^33L}0>)x^!8(RbDm3CIeptsboNf_c_0a3x?64VcAmTUQ1U-Z{begi; zkyB5)CTYFP!Y`VW4m+W4`RICOZG7{+|3!R@n7c7c>yts8K%^x;Heyx8G+f-0ZSw~- zj~CqHp%oonwd9DIy61FEq6+2wWH{^~Wl!r|0vWm4f#kXjw)Bz(Y)5ENoET z;D{zNpOIAcrH|i#wf`ED|vZR+)Ri5kQqs$Y>?d27@WA4UzYbfv*Pp~jj2qqV=Y10P)shif$~75JD-lbpbD z!nm0Y@8!e}F8dZMbOI6AR^%laTOQ@23ZsoWD<@QZvZ8o@S9(C|ZqZA?Q=(kF3J)!5 z>odYf#p0IR@M~7EK1N08Yn5&#VU&h~K$N0S?`)WTGIsV}8@)Ej;J%KBtOuDb^(uvO z8)K@cR#0B)hD2IQ=6x2!nT;RkhDQrQEP|6wi*zMowe)l2H6LV2*Tp%d7%gBw>q${H6^5CCG91;U!t(Sh(VQu@#esaCw?bmndYs}FsXy! z5U7K^#xC65pzfXnCef>-6nXrO&fmjs5PYTXK@BbcDJ@q`nf{G2_prEY1XbcCCIxFW zghpodPB?`!n;fU+Q&IQoT5ewr3?+f{Wd|!-u=|`OI7Tyv_N;PiS-8V@15@R-Y4`4s zFtx^^(L830;ojKLA_YiG*?G#dma71RUX?D83^>A&Y9c1fRe&Gfj6GN0k6{J~5=AZK zb%F;+teQZyI(`wUfGtD$QR0>@!22#6Q*5s9)Fc4Zk|o~FpdiU|xUl(n@+xz5T$?S_ zE5GG5Z0-T=6K6M4A@AkQ5mJrPp0X2U-m2TIuarF0I{buV1>8%Q#u|8y_=h2ueD)LY zYP%k=7a|1#$SmsqbA@cy!UN}E*rkg$G)fC`OIvtHrG11#IpSt<*=pq#8{{7kIUhxO z{ZgkGM!yyx?sZI&<`s<9gRN-GU{QFrNkvT^W505|GeUe<(}LHmw+QUs{_5_(d?gg1CQoi1( z4-79xxDXO_p{JwHXP}mb-+f);@Bdi>Y7f2}j%o51Mw+<@rY>l*0+a0svFABft*gH# z=WKq5WKqQ_N5$01ef8v)$~2Sb`5u?u3z7V7;2vB!)z@RuFMCUBbhe(X&ij3bmF3+1 zE3exp*8C?pEyfq5-yoN*PE-4SshFi!b7!GS{{r@_?xr5) z&At`On69pf+lA@_aoXy9!HCk910m6vS?=zU&bP^k5$fg>)<|zvFJwK~t&k#sQMJ;_ zPmiQNy?$E$#c7R^(D&U>XRao3+xUl3ceVbAn>-C35C!1 zju!=AWS#G{9O*Fkv|LOa_Wc|V+WJ{J5*pcAIb>krSvd>&^IOte;8z4DKZKYl40S2r z$50g0AzArIeCs#H_P7HtEh&+AA*=Uxs%|je>b|?2#S6Pn9=EIM} zW5!S_BglQH!+?-ng<(hU1F!p)tIFikM^iKK(k6B0P?t9F^7}OChg(HMNx15dcSUdU zE0gBz8z?dYz^^EYQ&$L8nD0Lw70<<8_99VRfAwnjjS#+&uT8~1gMWG@?P@yXAIxF^ zx%`D~3g7Lp*So}|N`iAxl;wNFq7OkLF9ZVq7m@U3_2g@$i(7({qHWE3o=6fuPge=T zeFhnefU0QibzgCH{5B-hdELSS6`Tr|sG)T;I;QDs6ehV9wfXP2J0rS~sY&SY#1)T? z!E|HnQKZ?~kHo~zaeM!Ul|4tzPyG}FbvjYS-*RNkCvjX<(&sWKu04D+%2Pa|Y0EiyX zf&3~b0x+idjX?f0%-u-wjhT|Tyf}ybuZQ$ZBY2jI-~i_H0Y1xrT#s`A!v9<>zW{%5 z{T96S-x1P3i;V-oES}|>|D+KB0dfL16Mq}r0tVxMBS-%)8UZ_C3j@^6uV?02HT&7= z<4-mOfIo6`KYPsoA{_!?9{ffezko}zvi}35GvMN3=3qDAF*e{~F$STm00I)OR z<}&6oGG;aYFOv2DL#`fh+Ic3EaROEMC!oU)pw@uv%P%DOw?L}@xeN2px@QO6Mpys? zr~gx;K1K12~fYu5JKxAWQ1Bg=qJ?C#X1nz&u%YOec3UEa^ z{#9j#=XaPmfSn5?H_*-gWF@czia#8HWXvzK5&+wT-{=nij)}we7su!i_a?w&<9F1U zSfANa0Q2dYi@@=RcqMBt%^}X=`WyVi{~8S$fK>2TJpU&Qn&X*C_pcTS ze__z$FLkjdx(TJHi&UZymU>!%Ft@L$}so+&bBe1|RVQ znr;usXwccoi$pw8C|~ull9C$xW^kiv`XHk~&uyW=(;-Xsl|tFPj_FV-uOk2Hxk9FQ z=5qeiYXk-Ph$6b?vuSl#O>hayH%{rV4=W2i<68W8#HF$#EYuj1j%D925#8_ILb`?#K19#inxJQI7@hI1@D zzoeOlPQBk2IW+uH4&7ZiM+OPv}%3{i*l=)6r#_tAynzdi-K9EM|Y7K zL&qM;@K6m@fz)BW`;)K6>_Lqvx#XsjF`3pwH=%!y3mDm+J7wGfte{53a^*;cb)4r_@W) z50X7KYQ%RRPWSvsvnT6ZD2jdrP(698Px7kGp13>fnw)v2e@V*6-_X7?tLP_Yo2~6v zQ{{Es;wg=3?qB+tq^d=4y35LB_4Wv1cE5Ta&2$UtwCVkP#cL(0eA5aw?;Rcy&Uink zhJ*}U)4PCfSBibc9{!PcT%Y2;)(CX{Y@EOFo0%}YIO4xzsI)-#(r)(r!W9{(p$y6D z3xVHY_bmv#C*OmLu))G&MxaxstvXlpbtU4JuS5tUjFWmiDy|zn^})EVX*T5=DZTKL z_@SOpzx=!9Lm0Vrwt@GZ8{Ork@n_DD=m!u&VyIH!jC2r4ks^Z z^TqBrt)FJ=D!w>{;gH=0n1Zk+^x}5<_@;LkFb(f07uU1=M)bgi4`cN@q@eN1+gQs% z$Eeu;q6js5-gA!VgNvtJwM6BU3rrSZDC>~mU%8G*)=x-|!5drbCB6)2G3)7A>0LGQ zj!#%t^qWSc#*f&ZO)nYpVR)s}Z=5hXQ5cu%*Hx}GQWK*{8ai=CM-el zNFBE#*T3DmH>j!Rz=nX>MdYm;d@PATlfx{Tk~Uml{KRrk37D zbB?uYOHA9l+T`l+6GN7DE1w|Ft94>@?GBriyD^_fr|lEA%jU)lvEyBa;WTOc*+5O{ zrOJ6cqPcbIi!R4H9YOlHghZbcG_~51IqB@$dQ6->+(Z?jkjco?iEFcO*?O*DE(C1V zrQ6DcIk)W3`os`^hGAC7@_UcfqFrV|G(7Yp;;VH#Kj=K773>^Rz$7T9y-d{Sf>S$s zvS4c7_^S1)cs${uN%A~t{Y0Qphx0@hd?Z5|%qx{ou1!n$yPH-%*O2v3c6%#Nv^a{^ zkcbfLN>RIxTdeHf$>I0C_|b}%BA%xTpU-1h@+}44Pp{nMX4IVTt&j6+%RtDUK2%9A zCQ!DcsM=N= z_`&XSw9V~p+f@&Ser1CB1NaUu_(Prh^vg6ouUwoLSk(PZDziV%*YlReV6Db-z^X

NY=ko?lM+tZoeK&VNv~^7r0iB%#!gz!2S+fLIN3#DnODx_-ZKw>PVs830ZP zf&8jn3LLX!R9Xx|KXC#T8?4mh_$4b_c#EEE=?9Mndwo@d`Kt65Q)l zf%Sm7z`8siJ^HHRLZsKAct6cqPiYYKRy&zB#BB@l=4W#cnA>#XZZfFxvDnw<_UanR zmsfm3{ZU1=x2`vWQ|;p{Y|{3K`au^NXK!&Lm-6BnI>suiY5Bnx!)xr9#x77#aN#8a zxGeQ1ETNRY+5 zRz93P^Y_f0*SWc%d(G7iI=b08n0NZfy@7sjdN0_-wS{Hd@z{O8%m~ZEvO5vX5DVI)r1-+u#@4eTX15NM5*jN@ zsDr(3O)4=AZ-+zOo?!PXjDwG{mfzL{qiw|H(*Jy}=Qe22R&!Bxo~C_o#9wJ&lTay~ z$We9`V3QqKfk@dL20bhEq@}rBhI4`%vL$Eg6rr^8`vSM*mf<$-xtn3c=6G-nND|(WWgA3?C$Jt z3f{rk%Sp9vch6?OQqPY`p_+|c&f|y}%XCP2tQkzoyzanYIHlW3c*?tCnw8SwyCt(I zt$9mKT25{;3Xe1Yf{7|TV24(czy%~@u7C&@DWPy2)CU13`V#dJUuv=s6VQV;xtUl` ziH)h!z?iVOfwtZgIR!z*#HNvHpSnWFPzM~qxuL|R8^{GgSAZqtc;vijM-ukZK8>)J z+Lu=9HWv$2=iX-xgw}s0;_>{llHvXi0{e2wrjM|Xe>gZ6 zHBHxzNMPY-&u@3m)&>-6*KJSwE^kyaR zw`)m0)5k%!lf)r~9L%-VSFA}4^^J?GAb!M^jqlb%JT6pM{eZ2(DQ`@tJvS)~naUdG zg;q)%9E*89$mmbeE{r-iVFMMh2#cK<^pKwDz#o4nRhZq%&4^b$(5NG|* z&REgJ4HRU1rwAdO`sy`ksj4XTYn7W`HN?f}1{aJcu{4EBY8zS2mmN=QWAbM*)oalY zhAKLDj5f;`D>XHkbm)49I9%sgTpCJj7hefE4XH~xG%An6x4%~xC38j{ytIIkT0^l) z=4DIoj;{fw%eTf)8EeT;8_O0jw!5VW8c)x?PB|k*OXxNXbUux=IwP+#o;x<`@7|* zho0cPO0uzRmf)O1TSx)xY8dzHO<$s3u7P;OLo&Bm)L6buqB+s`_=aE= zu$z#`L*Qs0<;#)Z{_3VW8$+&fJf7pGP zrc=K_b6ng{kzd5sO1jdV2z1|?p@lf_jC5VoT6^MsLMok^b>eAotiBWtXmlUxwM}z7 zTB+$+o#WtXt$K@mb%>zI`0Xq1^=zj5&mwi#5|f)`cT2|~?}%oVws~=tBtPEJTSd3W zA7p@!f7$jPYVwdv(-FtX6KF-wTvieHhA3ynO~wyg$|JYZ^mhoL*05}%x)|~;DaP=i z;^mYLDTnPx%7H6|$yxmVV-U$-a&>V)B)%&=m(m;p!XY3wOB#-ZnkwXTtHpmJd&^IQ#&B?5i}(n^Mh_el({;>k40qauS{^}YwHrHu#&aMwD!LLw+5G%tCf3eWgI{5{ zi~mhoh?5^1ks~_iH9@SL0xy|;f`3hu?`7Lde5HU|WIQj%cJr$233bHt;0xoI{_~33 zxhtcF^Um?Z`fRVM4|w7Z*e*DJW_Ox;HNGrz9+U%p^X)LLKfM)-5tHUixW#NP7S*tC zuV~(RgIy1vZRff+CQgpr*O)9XWsieRZ|eow_^CsIQsm_)@Yfe6{eFa)Io}aZi|Oi6 z63$JMSKT+n?Iz#nUBc4)LigN3SN6TnD^J==g8!g<8H^qYfrRHI%RJK;paF&B0q^~B z-DciK!w4Kt+>^Qc$1POMI#2P^532;sz8RT!?1qs_Y1klWM$ZQ(kf`K)HOwL^f->wO zT2}@3+w#g;2VZ2u7ZrBvmuv4BLgxpa{aWbNmWGtF4mtWH2dX@|ZEH{0l9?y&N@ibt z8Kq{MF|rZn8Nb1!Y7H7oewnO-je+EGu=bLt+TnA;{2UC(jG|FV`xCOc%vqfTT z^y4WIF_NuWvkkmx2FZ0s7cq8lPO{spmMja5)Wq=CxuOkn3s1hh))D@bQ(xcV%-cQQ zS-8C~>nCHaFbQk?7g1NdX)h|5X6xHAV|}is)k_GBDpql8yA%~wo2qe0l6ZL^?hMqn zIqs_lhZlJReLvDpsXju_UMPDB|=xKi+y7P)Z6*WkmWvd@&`(Qty0D>rhiJh*Y(9u zSZgp1AP42qbR>sJFUQo!rOo6pyr$Km%(C>nMRQX1IOZAH@Ji z{0(w$6ucJN;w{*^q^t1If&YGv#u{~KO5@`z~4i|wYQhCWYn#MUYv_*Brg*PZU$$1m!K5vyfFKy z`xYO}Mpl$6druI0Cs_^Wb13-L3U}3N6$O7bYOPHwxAoW5I(du*&dT5jxOR!v2=jLz z4IoKg{R}iUErd=?7guNA*bQg{?O9OGL&;O9s1aySXf8<{*3F)t<;bo=D%TYAA#`8bEt?FY9!qi#hIN`HB0c&9tjm#v=^DRE5 ze0AlVa0B=7Y@8Jb_y;Fd-^lM}w&YA|(?*vgExOBYS_pR#1{>T`)ITSW`P4AC9njWh zH#GTc2R?E$c8d$m8{`S{A;a!iK$Fa7zUxSDcn5XL1PW|`5{CC}9XwKZx#PEzV^1Tl*R|EoYxnq+y-JV=~8 zRZ2N4QPEb2nen8VeZ(7CHaREaQzZQp6wy|Oi7wGrD0-@G)yLRara0eX^fTa7@T|V_ zjAg9To}R@q2t=Me9;0efy`Du~ZeGE?Mv*`ZJmc6O=py-W=q<#&=xOTa3pqxXdI=jO zslrTkFtrw%0aDHIyB#mpD}QCA9}q%Is(|<`8TBlQs(e6?`w1#va`dt z4+F>Be5xBS)H!rV4f=dE^J?VLr9I_JqnK3*{oZWib_O<<;WtFi!m7+^*Yqx@*1-1G zo7rShUCmnMfc2M+m$D8C>UfFXphxth#!Y?G1Hs)C(di( z4arF}U0msHQY5dxk{*+I%Xe)fhhU>moFueQ3s1^fXAU_I@%h9zX&`EkD$<=AZVDxH z{60oNnD4SW;R6-7fDyZ8`3DofSPZcB>90pCI!4Bwiqe28vB(DrxCvlrA0fvI#2k z2JD1IoJJZf#Fs9;KT_nK%6%qZdgP#n-^J4-5p%H-5}#W;)cMLPR1E*?b^@Er>?0Ve zNx*0;6q$u#srqWq`{_a6`12d>5MDb`9O!5mLNdfTUVKp;iLSm3e3Uv-1KJ z*Q4$*H{M$_O4QmiDAaRv#E60ha8daZq~T>B@Rnhl@_&Qw0~3*d-C6$MNB4oby}!QB z--adrvnyd6ar5_X&fMzChQEHt{pXc17hvhk0RRjC)D`Dq0VXE_#QX2x-2WD0X<%=` z=x72cJUCexIU1SUSsOSo+BujpiP<^WJ3GpV^Rlus1He5G56`bh#`+Ax0&vT-$1@LL zpuquP0>Iw_fPVlr2h0-werUgiIsVHKnU;Z(l_L)e7n6vylb!qug58L}+qX39H&(<59zkzwN0AHIM*ez_&KHAS!25?Q- zpGT;H>f`39_>QpgR|! zQuFsK?7tMq`~MTtpU5Q8;=Fa`h{_+RL3fVszCt>FKJsD7;pU{Ro3q{Q;&)8tO0FRG0px9nVEqJ}+I_^FNnY*c zyF(Y<4R%pTw{2z0h`pgRX1)1JH|m`$46e6|V3W9v97VNDD6U$JT+_UCNj#@i^eJ9? zTYD;&@o>#r+mpq<*mGIN1A7#eo+qad5k4)2JF#$?mPLLgaugg?lRXhak20?h*C<>@ zKV5zOvb!#RI>m|{bC%HbqV0l~#-U2>$haAV>q9d;eI>&|m#%Xqz^ai^i36um3}pp1 zXQr?6#T*f#Mqk{!Mu0nG{@xyT+`}>mj}7*VxNKOM7F>!lV>B{E z!#jh8fO`6mC~x^rx7dXUg4%m6r<7qO)04!@v+1X|uYuPp#cm#$CNX#jqbj}o)B_EH zE=scG8nR`)j6C)#&AH-r^G-0@HBz2s1_x3_G2@&jw6b1-(qX23Js{h7r>Pyvv6&&m zn>h}-_>;ui17+qJ!XXLc!N6mNb`6?puW<6Ghr?d^FuP`NM7fH=BBsCT}wY688?)+rsajbaX~qeHK$ujs!ui%Vk87rzqv! zcSt|81V+_x$JI=m_m31fTqSjiSIzTe-tzj(eK~vIO5bMNin-SFb*)F@dYw_*dtm0+ zZmk7|y#Zc+8FOe<$}YfT;VAoNpV9t4#PwFV0ntWq#IoWGe)ySWM@Z%OK}AKpd~4a7 zi(E$y@V3gDjSPF)gyz#cYvd#HTxJCJlr3p?IPMIPJ!fl=S+cLku_?rHU(>zdQv&Mt zd2fsEm>J4r?pFzpvucTxvIY09O}!5&k3CN|N5p6H3spvWzESNW4sb&>(T6x6@GPQB z`0Uw&kG8?51+;zWGu_=exqa9_o7tDu+Lhpyk0|9JTz^>D$bn=#USSwe_oh|fGt?wR z2HSjr*Z2`EZFAzVf`aGpq-#bEedPOca|WBnLn7LjFcoos{I}h9T;)57Fe-jc0RfB; z(c!y^S!@%9G_5zQEBwLg$Tp-8Ms7Yli#gvq)fxt0tZ=EG9Dm<^*;;bC^;zd`$70va z7US)QYmDl5s#p>0D-8vsl{a1FawpYuU}VSV5^7};b^?6E@~F@O@a-kTq@<#aByJ6uqGdQg{gk zDt#Gah0VG#SWDsaaJn@_G|-`qn|#7Oa0dn2vvo!cfW4PS(nlNQ^(XF&K{Vu197A(N ziNoTiChQU^2rL07uv0E^@_@DLFd5l-pLGsHz?V8k{Bb$?O+OB@FKa2cEqYu^Wix}o zLZH~(Ys_p_PEmX02Xy?G+#1qdKi-WakEOdpwb7Dv(h1s?G~0o#ib1_Fg2~peltu(G zqlo|TE9}aX42@pqR?&QYHbUv!7cCYP>u1SphzJ2jZ3;%#3uf61Qx3+Q@VUzCsFg`= z>+>Ar?TK*Wh=-c{@v5jq6mf0+=Kytt* znFR;QTNuOs{sW~ib$n|-+t;>t6O}UqZ3(wQ#a;L14GAl)+pm^iEXuTJsRbJwe^4+R zGK7yFC2jMS9CDrxac<9*1+V)0rnnjpZZj^8(}`*njYI*d8r+F+f{oiL=gvN$$6lwW zJkf9HF(*s^Dv)aa> zzOWN2m~?ojhH`H06DlSX_5FMXHz|U~eT5*vp&jRNB(M-J_heF`<8iON&FgS+e{0G{ zY!RU+)|0S*t-n1GszK(^(-eI19#8tC{vgEeG$~EqxY!J&_wqJd41KZlb}umT<$S65 z5aRQY_?2`R;bkigR~nC;NY*a88YC;i4ro<6cE{*&#~%6Ryy+vCV&JJD3MWGVZpMxk z=!d{oQU84xWyrOC)nLIQPqR1t%JRKQTpFf*XY? z#fy>m%clc&IZm)qem?qy10|YzFgRM$yq4c6NNEBTUTO$mGxVQ8LIL-J`4o+YUP^DI zso}(y*2Y*SIs#CPGxy}-xRZ9r4|*IL4kA)%+`!mhvj{+TAHGUc@hybONrlD!(E+M_ z#H6QXX=He?$y9IgLs2E44v#@6K8cQHp(o?SbEqo3zgxeuVJ{|K^)Gjf+|=@W2-j)~ zjS603_W2gy?3$~Dut3|pi)hUpN%m4hy8VqQ$r$ufA}e9kiuItyTFVgWkA&_FhA13} z1k}B8>gikvf~R*=QA?5g)>?rIAvtvHjvL}2I}+=}9}6g_g+JQm&!T^P+%l{w%M{wj zYX2CeOv;!T!C~|!6y^5)rm478kZt?6bp#Sjzkpadwo6Ac8Z1N{=sF0e3vOHqk>DJH zD__F{Ds=b>)Txx3cy^F8X3;6ioGXO{`8m3%Fo^keXKpT-X}r^nU*=WT+a67uHAUB$ zrEl$dk~ZYQpianw>2`U%1*PmtE^RYm;){A7F8$m5hcqxqhlH706Q8uQ(~E75#uAI= z*RC~Hx}p3W8%@?9$83}aG5efe@Pl=pb$cZozzTW?hSc)B%flyz3w+px;3Rt8uhY8U zO0TooiGlO%9#ci-zKv(x!V)<}GriVq&6_2xQyR}hPx5)Zm!L|F+x>|F`kjvZfg;Yz z3!B^9MdT%DP#yIMeDgWyi?ms*5@Rv0?eBLcKZYmO}w^GnKRi{er4mq;it zvdPwp53SUE^n6@v7$Eb%tarK)8)y=D)oD1I#gE^Av4-+;o;!;^_>yZc=~uP(js6ZH z@&hw@`uH8}od56__hE8%pH7gV!^qf!#pZYRv^zFe^LpKztScPiyFQEB!Os#G-=DUf zv4{l6hIT)W*SiS^qiFkdhvm7viLeoUgKy|0M{g|hnOOQ}#t8buhwp($x7sJ0PRw`B zJVuOCL&7EAiYxXdL$#eEQy={x zIu;N^cDPVYvVtSQNA%VHlaJCTZQ=VHem)LQ8LmkW8k$jz|IWs&BJp*wJ*iI<)?>;3 z{OaE6Frvqz3^ikJ+hgnWkO8Cah(UW{^(pZWKE4x)elP213qKB2`0BiuSASi$Tax5# z?;X)W&YGb1BiV5=t+cWu-Z#EXV4BMZqUwBn`42z7v2vT5DM`87tHLMS@~Dj`?FZT< z2J$Ln&cWAcap&qJw6thcQ1G2S?&ViTS=B!#(3msS=_NNh|+rx<-Gq)R^RMFpCOkKG;<6|5>n&c$T%;LF=K^YCq2!m zbKA#5otgG0yJwONwO7PE8eT;!7{Y~X`m;O4)$2*8ga^5^CBKHe@4vk)Ze%y}qi!J7 zY;j+!Xe$q?(P=9k$%n31FNHdt=<AV<+k<5lYq8Qol zqi2+@WVh=VRq)=bP<-I+GR|rD24>CVZJYJmNt~56qOtCWD z9mv}XntG1yaaQSCrC(x^QD)t8uWULUN3L|DZ`!pdDz^5&$ydEUl;A{iuk~eqNa=9E zAiA?Cbzj-=F+`Q*Sqf|)!bVl^Q1TgyQr&jd#QiM_o~p8$P9!SZ-f1-xapii#gx`pf zjMRC%o-Ws~Vk;wC9c*T+*caS*Jr3r4NS0a3Os0(5Es&bN zZCc{?paMdPI+)xis52Wmt~%R?BAVkJzE4Ug*N(hMSB)BY8Dme?P1P8maL+PgfGxc3 zK2}OY;U+FgnK>Am!n6#kg~DZ6zjc~{cv!#V?*tK)9=kE5QyA{THwd|)*WbHuqi zE*@NzhNZmC;B8ykCLM8eUv8?Dy@E&D4{QF*B**SkAK=h%VzTNBR^;t0*gclv&iYvu9V_W(C7q1V8?6LCC4*V3tMILq}_?COg zvK3Wjm&M^-)m+Jx+e-UlT+fkT(xlfIYlfXR; zAvt^KC9OQxCltw295GR2oOs_1L5X~5o;1|kHiY<;UM^Bt?nq%2NP-qW@XVb%veVMy zUT4}3t$s)M({>Bj<2#m|^OP=}GhCFs&$WHWt-ZUWU{75xTuBklsOlawpFuTIDwVd( zE!s%TMLG`JQ=E*`sNqU9zgEYn9RAGu8M#6K*g?-Iw=Pli>t{Jb7OGxn3*M%E;D2FR z>L+qJ{gtzUQEWjD`DhF&raL{KTvVaC=yBrRY!|#?Q@{IKCnoI8Q z&>Uf4z9QH`Ed^J1ORp)b{bnIF_NblDf;@{%Y}h|UChyHH`mj6Xv`7!%fS4CLW=1Ok zY+Qw+g}rqbe8^cA^jv>dA3^-MZORg$X_|F3FzFd{^;E^`9Ze!m<32O2%m964jGID% zaA0?>?)W2~{R6GfyB64o=%=-{uCsP8*u8nWs;wnzz1QdJ#mw(cqgAbGXI@6^Lu|*0 zQx;91{;Bin){4RDK807$j|=-d&(A9*NT8Dilo_H{PyW2_-y1hR7VrQjnn2{f?U7>W za^k@w3#%_x8@j%=cdw@2&x>zn@*kUU{~Y)^dWmy>wLYe9;r5cVYW8Zu%DDpwB9fC~ zhX&|L&+aPgO=%%}+`CGARzlB(fFB41Q2f{8_5Z#f2t>^Nw>p3A2l`)Z+YuH-n_8ej zG+QJFYl%Xdnjps}|5vW4|L@An8ec#xQxw?l25s{KTPQ*9bKnaCu{r;q zFYA3;|7FMq+h#Evwxpz<(RKNBg7HFIUhk)innUIj)ec2pd7_?PWojQp6>h&lA1Wlz z!)=}Sb8I2*68*7b+=qPvD(}Wrq@^#91&kPr-n$fzTB__xcWAh1#fH`_EBv<=yRd&h5#LD(^pAEe8&VPF2{Z>~iBC zbP@b9R^Q^!PF*kU3nO%u&_o-kR98;B8f@fS#5B+*xxAF5x zGKVkkt$y8<;W~NlYgS%AQ?vT7`3RX~j@g&S}}U!qTnoPZ6LZ=!>QgRR7Q?eURB*Y2MT%bm!7NhP!d7 zhjukfugU~=ja|QJeQ?$+^QL;DNSmX`?$8F_-AMS?;fXH3n~a6-v5sbwjD`7W^374- zg62&^vUfYDrC(5LUa(-`;SxI?Jy!4Dpqc#R*2BE?8ePnCPTyIBlO1%zYo056 zW2hnNmpzj=C|dudlk&j>=lIDt=Q&VJ#opWsB;;o>)$nHpNa4rGWrnWG3Fco9K$QCQhD(h=QbYMlP_V6 zJYzK}*$;M__N^c>QRL=_uAn)i#S7-W>=hoINUTo1=5p$eqpvxYsjl{(A$?A|d9`P9 zqu)BON1XZMp(!E2eKqW;ypuVaQwF{(%JF;&psNkJ^*N6t!?&|?Y=*n*ibvT*^|r#( z)BDTzPP%rz|2d;Fdg)VTbfReheMWIs8`;tfFP$DNkP{tPov=OcTJ-4@SFT0*PcSxG zzdQr_%HZc5MOD+cR@w?2Cx-V@upPWt+#NY&UeA)NFn=rEAV0-q zNAoPvDc*iXlwmgc<>!{eaV>`z7In(MzVSA=Qujo_N0q68Q|Tr5)8s%!KID7L8(A`% zsu^R{7N@M3PY;IEcWZjiNcGi^+?uJQtsxEjl$0-pyx;e_;IaP%^#@qDxx=bY%)LNi zS0UwHt6s$EN5UegZtz|ibsp0^M{bonel;eqiB%zqx$jM%(yIQB+Wy|A$do<28gwV9 zt+t&M{mTB9emY>MxZYxMt>nuU)s!8L2Ul`t=_Y$^zwA3FDYfU!ohQ@RG-KJC$-l|O z3!kFv4k0D7?hZR~Y4!Un<^9eRJHmoA&+$979xOG9UM7z1mN^(le2r>PnvE_CqW&Rb zQ4w|c!zE$$z#hX$o7MaaFzdr0&YEf=`3=$rv#F9WS}U{L*9z|mD=CK+$OaP0K51|W zZ{e|R_bPnCt7WZ9$gbGAW4i(cU&aFIscRI`jIU^{GYe~_ zb+Vpx`iNe+?i%)f=c9_#-WtWz>4LYgmLAhqeMyr(9G@abq(%iv)rCTD!(jKhPs}{d zI#?X%t+x#oz(7lGk-?L;92JSoBP=g6w*=2MTjSt&MchC&RLv1IaHt2P14La zkl{JS={A$3AoSQgwJs6Q*uDAQA$k6B_2G)PZHNIXwELk9TG(;yq?>!hu{v)Zw?_WE zFW6h)yNkb>s`8Bmmm#B%@2`>9!qUjihH)rFW1WA}T1uEqc1*<_URs`eRF^Ae%Z{CI zHuUH=QGb4ZoU2n&MAA8whH62L?)JOHuCW)~C7JBf#W{DJ+wH7NpYt=iJTKOWZ<33Y zE+j}X5&vusj7@k?|0MXG721fbKbo7IseZ=-}g1Q z9*^q#6x^KUWB6Q?mHvcvPeH;i`=Wn9{Yf72Auy0mdEQkFWpvx2mdbgnn`7#_Y>uC$29v?d352n&17>%Ce=k{O(SJ-9|iu`iaRUvb;AQ(gpX| zhdv87(<<6&mHSA9xB*pr_g<+?&2ZYaao;mc2=$DU{Pp|JDyMu7Wc3Q0P+%PQOuU~( zE~b`p6g3?8*r2PQU;c-ZH@K3+ZZv0kWEBWyjOzWocp%IQV$1S`aM?OSPCCH*&9B{ksv{B7@jMdkWEAfux%aQ2)-7i}|GKalDuw3@*Y413wbB=L- zXf^$=btwL9Qy8&+Nz(t}Y*P?}1}j*g)j?`7_z58d5+;JnHKGRKEDWw<;_7PQY!9A* z)~J9dRGb~m)Gb_Ldf-&%1PrcW;pqzSG}w)8VQ=nWdX^F__`{oH$-sC_Y^@xeK@KK6 z0k8xLt`39CIcPZGiZ~7ex?4Dd7eb9fK-gRw`~kfL_s2;XTpk8jvoM3o6chv*d!V&< zASZ_)6c_-CSOW^gh(K%C1pgY?3xfq=|4sz|6WEJ@+$QVynzx+MbLcPGd7;OsulTUk z+d<6~vX9AP;R%NCw#hOXb676j`>4bh!RN)_jvCFI6ydfA4>K3L;p)B=^_p2eVR^BE zbM&jp2`j&qfV+#U^Yg*`dYn=%BW@@mMascG0tKbmhQ}H&%x&Gv<9$zF=(e>juHZW}lSd+X&St)s!C>UXojo88)#>SCCpg zLf?470&}Dxt$0>zZE9i^b1-gvCrbV){Fc4cq0)U0l-wUA;o_rMFQa1a&vI`wlm1FU=a<(a z)`MMs{@+)ALY(x6k5d30J``SGZ^e|V~VZJCY2 zC-gD-z2zT?`D?nD+7!z(Zco{5OH+C@VDQvn*kaMMhQ+11n)x(be4x&)32lg-`tY2C zFZ0amSVcESx>ys{tg=GJNed&hr~sGPJ0EQoba(7JU={d=&k*CF9T0AJp~rpb#~ZD- zbl%r$$revqqIYM4KYI8A zef+_g`6eRs+NiMtn#$9Qf%(g7(Xp43b(K^Z*ovH*oaIkkyv_79gg{kkX~po+*OFo}&;8)&JA=OB@bFbv#^7TiCFw6-3+=BwHv7{-pysZ4$(xXuAsvS@ ztoQ15OM6Drt6s=_K<3=sib^~$5O+Ce#!hwlSj5WoxsX#0L&iKZ$H~RSsAgy?-ql5g zU6d#}s5H`$xAQ2k41efrnI|K3CbD~+D38BZn)wjT@1|jX5fLHqlg7Ywsu|Hs>cj7G zi->OQhhjl@&d+NCR|B+f7I`~H`S(3ia!d*pI2j< zonJ9=Wd(OnIn+<}PKP8ctMq;kcuV;%0B#3Rsn5$1cr}IIuyBc7*LS6_Pwac0m581V z^``YZ-Lm3JG7D-@tJD(6I`@5;?cxGKM*SB`SmzE7}kt7Y#leoE2~GMKskoJ7oq*+A_C_i(}X zWQuEx%z>fk2BhozRP%j23Fv%^7N-z#sn^jHSk0bJ^;xgetT(3PG7h{Xu%9u>F z!Dr3Ug0Hu!9ocXFROVSW&`m9S7&eE%{umI@G?#yx@815M!6?avr)0)Ccy2M{5!pr3 z6wCW$%mx#Wf3P9^8yP7FPd@h3pdwNX_kJqSq4Htdpx4}p5u4&`CqA0%B*t<>9+x{F zSC-P4(x{B;mB03&J;&3Ps+I6#f6s-ig%vL%L1S}^Z>fG4Z`K=U!VAYarp#3*W8bHm zGEGTP3;di^QETv0xOVQy@*t7aa%*o10+B(>emR7@*@Jz1m($5t(J;1)4t{y6>ETo4Y9R^R0>_k_93IdxjHY!4!T@_I?=*v@#jGD8N+PuikOU&@iU zXyiRLBNi6_zVEd9KMmU!0KBVQ3~@upf1$rlL{&o|Z=2ZDccV5+j|PCy8~= z$MV)ThOVwb57|(EdnNMtG?{yMiArnvYOhNVdBt8?9Jm}h%S+ulN=kie>dw`n8=iC- ztp&^O@$L=ur^%Qi$X}RaHA(U=oHfL_5gnE*zb|i~Jf{K^i|DvYj8qhQ+3Zg}MYMlO ziJ_-{yd@*#ro@hi_t-?xawG44h@G8GROGM2Mn?P6lsbI2+@BF6z4%~?xQKe!eB`o# zMzf_>wFafq@RcjQpUvM;ybYnq)07#d(9*Cijh7vo)%p1FgXhF*PFQeD=tFz)IAmKn zYoU}-bZSJLTiDZ>*FSQr_hofjoz=Fu-urfYt!(J*_RsYX+HbOdgl7#?k{A=I|Jd$i zzGs_Aq*SKhsSj*lTB9FiBt&PRwC@hJ{N$)wU!$ifmkFlN(ZRIj_54d{z_;U$erO(8QPoACg-=}zU z#VhfPkU*bYMefhOZYJG*XQE^L^HQgGOf*K-pngU>^muHe59io>H~#TyH5iO-FXrQy zKECbbd^fLZUuJ56?cRCY@o?azdvQ+ehhf;B=vWEW!^4dek_!|s_k>kBK0eo0=4Pii z_*r#dpZf`UVn)lOV>4YJbFF5-{zy03wHo$A+tt`O)WUo>e|T2T4qwf$zMKi+KRj2u zgQqV^{}3V9FL|FF{5Qqc%b2mAuVq1}onc zOGE|)johCwIm?iyT_aceTz1DjVYj9W zIaU40&c-R~c{GL(8`Lhe9WdlypqrHEeT|vWuM0$te`?8P-80vltnMr6*~8R5{dNT= z=eK{fpth~%c~|Pp2U9;N4P`b!b_GszquH38I2%;sT zXQ$r(@K^ikL}ExyRs7s@PY4bFxz41szGONRyD)qX-o+dyZww-ZJ@-Yq`)aaCPgY{Q z9qCz08yDs#CQy#u;AIB?3!pbtwAG+Rzs_v(I$|EqF$3!e~ z!#iKZkXxaCdgcg1wNF+v#Vf&&viZE-^Ky%($Dcmi2b1~R)E0c19De?3;`M&pC67-W zx9OHv4_~&Ho)BE%N zsdeequK2;Jjw%UN_V%tXUMPEQ-b0!1kvu>9 z1CNffloj89NY8y!{IEGK%T%T)cKksyI8FnSsPZ|gZsbb(Rn=cu{>^0y8kVABgb9AT7;m!Wf)Zo%^Qx|ujf{7FZb+Q znXa=s?3#96^-jsnZ6-AjcOE;Dk)It=*UOD+s-8HYEWjp zIC)yZC#8PN%n!OJ+*XEBsc+aU#9y8Ci7mbDd@8+EgIZp8*TXlvAKEa8LLH{7)2g#c&{E( zRjgO87Q#d!?j#dUaZWOEyO$Q*>&Vyf-RI-HV?`;o90k$EGSl{cD>mMVz59RC6!XSQ zp8gWd@Pfy4n=}h{Dta=@TaC-m?BItGwJ7VeW z#_c;%TFyL6FBmVgeVn?SaFO?|cXIOwYX9e@%##tqiXtqT!T!^%7&W>WSDo!?c8qiS zmomCJ1m0AUNC?b3Q(wrF2q;h-r8i!>UP)ha)zQIoh|lyw6z$NcO7V#2k&tr^qW6p= zL;{iVkxEq=(J|5Yb`yttG!%xpKh69y%pkf`*`=dB#1mdK-+cepoq0Ip2ba)1yj4V^{c3>z$|8;}}o#EKol5=(|tnvyVo33i1q-tc=I6y+G4{_aeKN zZwTYQ6!GmT6{kVTLly79!AxqKl`B^Q_%KVF=UdW&YO>LyLd>w253+4Vn3&rKs5au`uKd z{nl-z`z)7EyqCL9<`~S!@}yAl$cQWLcHMo$5@r)^QKQk6@0G(0f@)`Teq0awp~7EB z*Yha0e0jWYV%cC)=HmHG3X&uZi?Ob*wjL*~=kedH-Cqv}wN;*Cs!QHMRA$O)+T3{4U(43gCy8abhTPpV_`8Uq#e^_n9! z?}$*J5mMfl!kOq}1#Z42sv~Wqil*N6H7io)P5=8WG24s*iB&eLUCjIv$8%E}mE;)Y zPKNOyiRt2w_Ff(}?s$h1j6(@lI~wtwmgaBj{pjjCpG4nAR5i?Q=CfZw^$6Is-(>4p#`|uO(`Qx7k z+F!br``Q#wN0iyvsgjAYuoAbO%#wQdC}+IDdYM#A{O*r}qIP|mRK%m6`t6b?X`ELa z`$(oWx+G^{8R~=i6AQ%q${%y7%sr-SH10p5%t;?85mDK6ZN*Dp4Q*)3{iaD@C}Z?} z$ZEyY=fV#KM-HVJ$c&qe_Rl0xO~`A14HkFIHBRr{Wj;Y&?J!5tefXdd|3QkR$_$U4 zOoq*8wZ6WU@dJw&?VTXbn-M0l3vtDYF&cb%?o-O5y@{-8yR^$w1$a%H!K5oI8 zL^e=i!Q_}Iv3RNVy!ka%QQyEhBV_Y8)eOHIz1aSf#JYYOj-3{QVG*S1-lYX&#bs*x zaguzP+!rYyeJBreX{Eau2=IJ4!#w_SRWzkMOYep_H$%>(46WYve9Mm)n+q6& z**V!S1&mkN6cst;kkbmVUU+lSNsp~K??GDEKH8)nDl@V2Zf3WZug_TnY?n>PFn2rb zk6##H=G#wuC%m{mI5x)~@c{;Bj!nE2N{ z{=AZj7yXl&bvwK)7t=l#70hXjyBB4{l)Edg?r}QyaeP!qxOhob=i9qB^Vvh?PSZ*` zoQwBHV+wZfI3_=QjZ>3sY2s(G+cQCCJB0Fe2i0dmp(bNanjM#N6!RPNN0u}F-(YfW zPmL*e`Xm@xvRURwm&n^WX9Tzc`NLfnm-jl~;wSM*f z+c{M(%RP29CFw!4>)bg;!X!Z^c_3KG#(**C5=-1vLEh88@5P=cbjrUd9AU{ii#5@X zKTUgr3_g~8^`n_o=Ctn4EBsa^PVcSwijMAkTA$U>pea*fs`oS90YgG+2rmh}XuCs2 zTx3zJik}f((n{{%a@Muv!y%MG$GaOeBYTWa=(iY`H)yoABt3FsxnlC!_}ci`)Sb!n z!%w13W{;nI<99%bxk*VH(GuAz0BfGwO*KlNb6U)iJCS{Ebu3C%q0;-9OvE09_pZPf zIuE}T2bZsIgJmLe2w}JXi|eK~TC)EhtwD)iMTCA~zW=Swd;NMV8DOXeMq?Zl)X_Zy zLkVI*ICUgU7}_8XvZ30$+1hS0nXg}#w3%EGvUNjA-9OpKT(5F|40y!7{BH?zr$A4AVe~V z&`*eZy%GOUWg`Vaa9trn)a$1Wo2ijdp&&rC5FiJx4+1q`kIy$#|K>RccqfoR5(rSQ z7yCa^gSTM7q9p><|0Qi}HOB~pZTx}+$^T2*&_bZWU}HH31JW6QWDx{P|1W7ng9UjA zj35jHvLj%CLxuqPe~BB2!7U&HB~bt|2oNB{13tOnU&tGjl0g_tsBplKKTE41XgUZk zjX_|*nnFUo2HLZc`uD66`pAEGxiCl+4Pt^2sP~$*>A}|Knl`|3_K(+0a7~{6bbn1^ z{&as$CjNAPt<(Q>AFo$`PjuHcP7vf6#(>Tx&^o+MB|t3%LVth>H9j>8uTKe3gF#df z1N(9SZ>(#Z<4{+xnY7jsJjM*?@ zArQnIP3S{Jt@Zgo)W4Jv9OC(JYBcx~Fu*85pyq25@elPno%5R-_(q}7>x8J+ekv|bDSndd-1 zY}sSLpbw&15gOGnn`Mnq*?%0Dz?5oJ+0YleNs0Y&Xu&Ct4O}BY4U`xXj9>(~hNnRB zsfEDY13{SQn};<5?G^%17(tM0!kaZH0ct3ljvzq|3gU`w$u+dF2pAm*4T>9b*Fm!; z0%Z#cg0yl3=b9T5#{GdB36LIP?7tyP+n=bhAha0an&ZhDz81q^LD(okLjAoV`xvfl z0@NT05E4mfbl8yd>krgIAPXPCNwP4WsNok424gT)Bv>?_sNsW543q$n5LVleVhLAo zeBwWk4I6S9{h`%Jz;OhKYlZ)Xfyv`0ImK@_3P?GY;P<&9G0z`rE(8EBm`@QZ8*fy@ zFB_VP2@=!=8v;`Pp=_KkAlPiYK@FefPz*T2W(j~>H!%%gfnr2Jvw=_%U^$+o;g|hq zcnrR}O=Uw|+&CQnSy(oN`1=FXz%&Zvpduuwz(%vFZ2X!-nXG_a1HaiD!rA?yXe0($ zf`E&f05w$720cQc+0f=ULbP~thEEIGLWDp@NCG9}jcEAP&=e3PI3q-jCusQ8z}5*y zQ$mgsgc4$;3FY0G+=RyQnvWGk-)SrNNE0v#WT#~@a|uZh)_DgO*Mx)VIwsL3wRC? z9X`*IPzPW$vALJNbO^R-DYd*KiRdgAXx5} z)M$`-7D33a1cJf=v=ALa-bQwn%R(8*0kN-u|QZ#NwIc zadV?h)IbS|0Mjad&GAg~1gJ58>~B1iJR#~oNE(PjyJf3|pnwB}j1Aa-4EFzF14V;q z*C0?Yp;qHb8a~HSU?PMDp;QU*3{TSVsRcpVSb|ax^5SmU?mx>47^=3U{v%Gqf+WV9 zsG-Vil9fLh%&~Zm0s;ouKO#&lh?=}**?*b{VNrNP8h*uv08SGjx7&EaLeJ){`T# z0AB*VjRcl!XxS=(%5N68we_};TS^ebp@!x`!jKJ@kj(&#dwM#{cwaRjJ^K;IMeq+;>h2Do1Q+nVFK4G2&R0jE6% z1h>NPGMw#o{ilZOgugC(^ALyYgpJe~(BELJAyD+@5pG?7VZeY6Fd%@_z$HisdEq$? z){ih-Q*TzLxH02*y|205@FxZP|EE zgSA~3kWSdTY&=O@H#6Frdb6mlm(#7OHxFy5|B0l6soKVI3Fo2wYp!jUwe>Flt!s`a zYyYE~Zhv#hPR*lgYI z&9b(BCcHKEW?5Sg8n>q2ENkn*^48RwWeu#6{U7igPuBh?l8WayzztP@+hcfsgY|%M z%XZ`W4c4QUt*JN5+IqaVHT7m$TaVYarrs=TIMn~wT*H$!NPPa9db6yp>+auh3khbF z!i4+|Q2wNiL+W}l|4oe+28L*0E5OGcc%Fmx-Tj*yS}E}d>dnL2df>4J6v4C%0Za{e zWsBf>4%RE+wRVHG4q%E!h$m)~k}=)W94I=4J!{7lhn5 z@*Kaz{6A4A9t4O#RzmP~*e2`0oy04NA>e6yet=xs0&1RN(sj3;R8 z^vG|PL1q&)Km-Je-aMwQpG2=Q98BfF>QVy4e<5cOw8jvm#&a7GkWw&*1FIwmH5<=u zupS<-HCq_+>k*Q1Jh#DmnE0C->_Pw`JPDS)S+j+4ci9*qDqu-Hp_=0f8a_3cdjo2M9LE#1b*gu* zzkoLrO~|z-g6B3MFnI;ee!}Jw5j?j6&VsSA-6&u%LISbCmvTI}0Rc%9{?n9SWTV@_ z#?sQl*}~q;!UYO31%q4KfWsEp8(?rJHwRY>7aMykTMHO4e83$ooXsrkU14w&J8JFGl9Xay&SDA>~U|k zaR6^}u{CkAhQYlpoE>0r2YU+`+|>h|xmr70fOAU+H)p7F?%>?T#uNN!;SNz-{3_bs z1|l$Xuyp{}K=MnQU#Gx(4_aa2;%Wn3Hvttp27@1mf^329I&fLATOGvWg28`<*;0VP z6=85C7+e_!KLvxI20^}Ha8(d~3m39W0|wWG!L?v;ZLmxn2G@nb&w!0nFu0pN2;k*n z=HP4r0}2)n8fFSA0)vAVLDh!Atw2*@aKP2GFgW0WEevi4gM-?DwmE{@!Qh}>fGePG zZZNnz4DJDggXVbwPJmZAo0wVPI0``aZfC++80`B$e9E2Xz@NxuImbEu?FbAE!F07fz25-?Z z7n~Fb1A{wU+QjkXuP(p<107u2+Qblmg@I%WO zfZ-DX69Fbmz+LD85Nr*I$=bvSx^l|I4)9#__z4{q#Uq+(+A1ctwkGB_4odvT9c+ON z$-A1^+L#@~eS-)fh^`iPS}-K&kKeroy#uO|vyG#x1CS)0-`xR#+=dQZ4_f~#>pylY6yv>{47@@&f_6w!8Zh9vB?Wz01-3 z?VYBchgLDzg8I0A@oJc`-Q0!DE}gm6w{{2lFe8ZmCE_o6iOLi(nOad z5?dmvrM}JgySl$pK0k7?=D4zv^|UL8gigBw!gtSEVf1)h+0!g8*Yinlj|IH1a?U*K zanIy&+pDgo%+Up9gWE=u-s#61u4H7EbtEhq35Od=xtS1z7LwCS_49kRN3TdK2MK-!DD!ftkratc1Ce@j6S`u=vSCEN4~i zsu&d%_o&716g%D zNx*Bqs0FH0UpjHf^r)!%&vu#tlJ9OhDO23hIoz|u;!AHi4eL%=I?BCzzvF`40YlDH z2O8+}xPz<{9z8l3*b+O*(($$?8 z1>*R`iS8W+c%qL2K2{>^SajzC}?SJ(FKKi9h|*U(^-D&sG!4}X=+jR z#>-31&x9rJWX=jsl=hzfQi?bnlUDVAj#dYU z*A05JYuq-u7kAMCR1boK--h)>E;Qxc}15^tJRRj!L<%b3i z4~Pyrpe4PK9~8R2_8v5tbwZN{=y@D1L9k}NpK0xEt+D+(T=o})3&T%gSFN7Y(JqTS`JD{3FrLX-$ zwT4;={X)dJR)SyXZ>U757SQ!IF07qH9D>f*?&Ge5YY;u|-M{Za6~l1`ya)GAfZ1{7 zgWrEP6p8@j_rIWYOU|q-L(dU;AsI@v zW`8MFEuCZ1$D}tZRW%0)dJ;x23Z zQj<_tr&21v;?lE2SXfru;A-wsNm1=`Z~dc3Glr8M6isB?e^V~7iTlRB)1l+bPqp%b zFRQXg7Fr)C9_Upq7Q^Y(OY8?{KV*Co8umD|T9Lw|-KOs}3Kx!#&Gl6oGCZeTS(fu{ zg1jr;_)BSvtA>kphxgRB<|6~72Q}Q(6MSDbemO~Y&uiI9w7fsD0+q4;+a90G+|aJe^!#kSCv9fXqHVF)micxoMlE26D@(a(f;p!QY`AN32 z3H_@8vDAa3bA zzCMMnKqEi2uxCxDq5!JJjsB3j0tPKeC9T(~kcNS@!Frtv{k=xB)$f)%6|_khT7&_1 ze5`3JFwq9;QV`Vj7nXoFOGEddErO6SR2Y~Kz^43Pzrc6{^3;I)K)r&xLn`)vq*8^z zjBQP&g1tgG20!EOss0l+xmh(duWD;licLqU}U#z`UI zX2C)rJ&>B-q|$$N&6-k&Sct3g8ZV*8!R8xifrJpC1r5iUivR7kz##z)))k4nINtLf{aAuodWgsBhOFzxo}j&#wcLjkSJ<`hNWZwE+jMHoYH*rhtsB zb;0^}ubm46!!uMrsN2^c(D$?cfVz9F+i~a6-@gwW4=OSVBRdmUYq+V2Gjsqe1>q?_ zWbim=;ff>teg4}PVB>6NYhmu-3J3NOI|#~HIa`QAg^SDY zi{Q-8&CD8ZXJZfWj-8vWtBs?r*MAuUAdm#!`5PgD<~TYyL&hV}A{QGw8(UC)9Ku;| z2Y_b#gdq!%QhrEm0BV6gGwdeB0)f;Gh~>;N-7}|TkGOiecmazLL6icO^~+B4$C*En z>2)Y&!?Mx;LMccDur+R&`fJtMe-fqd8av{;oA~|Za}wNYhp9=LtEF}o^T*yBX^p(2 zA;rh~kQ(i0beC1cIgHOX;@XhYkYmtHyKA*LM;|QYIKWSzHu^l-|E_bHw}O8%tGpskV*7A;g_lV2%ICQP^}fXN3r@!pzRy>! zo>BHYm6zVqUwS^K?81TG<#)>Ps^R;uoGL$i=qPbY{`CA3w4%r-NyWKMWx-B_?OiqR znMU!?bC=uI+lu>@4qhNFJegH89&58A+f&aO641Nc_F;g|Pfo)bn#ifs_W_OC&6JPuJeMc!@j>&@^WC(Lf-PFYi;|4z7NezdCwLk_eALm zNspa=<=g+tclGQ2EA*c{AAPdh-&4FQuFe`PE@9yFsP{hE%o-Eh;(5g5giHTK?@iO(NQJZ%2pyitZv=5 zq_7(4@P`UK4F1%bg1dO~X^D;t(l+tLaoU`aPk_qefSCDV;A-A>h(aBU#KBt zt6cRV3%|>(r@hK|E-85NqNap>+;_!avDhn4(^mM}VMzS7O^HAkW6+qEdD}o@l;qg0 zd%MLNS+F|q>uvORySLIjGM{@$g6TU@t*?6*Sq9*i=ykuUZ)9D3)5sv7!$lN%?PS8` zW^wb))0hD16TTu8tDStF_IXo_`sBkdoC)6^JbGPsr2O+Bw_j`Emm#sChzd3|Lzc`1 zx%26VM}|BnM?~FXo!*9`Y^9~GM0SZ{yZ3i|IIiEWU9O*TL=P^e!(B(aYxJgFu5I~S zpQ@z~tUI&)6x`onEYlvl_J1h7o$}}~!r{P}e)TgxE3pysr-#em8r|gord=;al)H1V z_8ij4%Jux41rJ?KpWaK_`wTxgz*~mIulD&WPC9#E`%Kd@w=(YL$96R&neON55`aka z9Lh)FCv$k_2>?8xn*TAHfaTDT0)?`@LFPs17leqy{X*x24lLLrj|O&e=pc0ezpjY@ z=OT2hJp<-_AX^9Y9~!KfhK_%);Le~2|GgvxstO$-_!dM4yj=f0gaQLIw0;;YF2o%m z;u0tYGLZrjpiZF!1K=g3dZCC+!a!9DgO)-E@D#(aV4(!GuO0*ZE?9u7pkPlJFgX+k zDjW+oEQ8F9073}Cgn>Cn7_7EMA^;wS_LkvZ1mQyp;fZN zKv4)o7XUWH0B_pn0}Up^&;fzr^#>XN6X-wzKnfj#p!N_Lf;a^+0u5>kg^9ye0>CXO z#0(k&G;3x2?!z^}0sW#OE`bUP0XcvGK3I7T%s3Et1aX`}gK0R(M*=({ppU^hbO-@( zi$sDQ=TJBn=zxNoU;qk(Y&=i`OAM%-2tZ@dIqm?i1K>;swSoE;g7r}C0FQBf0crv5 zMuch)LAgI3;Q6(CLZDyJpjCtpLBMvf?F>8%P%u;>U`&Aa>!TnJ;Sek&3%?9nYwv@8 z|Aj{zc0))*C62g(H8 z1Mh|okPYU)-w%}yo)ZQy1UyCvia?oeoE^;F%s?U<9y1dg;C}<*&%jc7m_CoStE;1n zC>)NvY~^g?Xl-NWBH-X`rO(T|0cEWBKmT@vKwlH&rULweEHyT^09oOHlBR6UujT4F38*xT zliPbF#GP}pt?BS??AA}0`B+DNll`gn`xlbP@cz^nTKArp(mx@aZ(ttYhOtRfzxVE= z6RSt$DWjJ2b1PRll_s|N`EU?VmSix7wPp28<%+^da=G-$WNB0rD}|&BV;1gQdfq4d zQTzm(OfWOWY#C$cq1o#+?^=}8ceqcS?XK?KRaVw+`JF0@XgsscSxPOKsG{Mk+G&?` z>k)F&lFp0w(~TBJ_IN&SK`Sr>_uQX#`?Y~aF^8B4NO}*U93lr2D*lb`)ih1)j|4X*hl@Z6}MmR zxyE*5WwbXdlZ0qTY}T8mu`kXZ`5zC}b-9PyJ?-|oDaSK)@1Z-_yN;;i?3b<;UA;}l zD2|SP>p(AH)$Ge*;z07EBB?tk+M(oBTu+u_i~O;C;^@p|nfuu&^$}er!;xKK7wDe` z?306g*Q)PA8i>b+?eQ|Q%Xr0LX`tIwW*1bfa3D)%+r&o|^t)z_dFSkb#Ijq|#xl*U z&X*@RIGChM@95Fo6rUG=A0-_$c|QL_O2M3hBliAm+fL8S%Eoid%E^+5sPN}io%RS; z!*caGiM>rDm&zs$L&e|BDT&Exn^JyIFFo-nBy+&4?8FsY`Hp9v0=m7PA!#!o4{HoX z#wpuBOWp5He1skA*gpn)wBR=N-h7u{@et*0suK~;soL>jind;XC#0wu+uE9rybW#V z5FzhWe#LZfQC45Jy}Ir|vcg^oZfi5iLR*-nyHhrY(J3898YyYXb4e^8LIPmXCa-ut zL_U!s9nsL5`hxutxR7r<$#Qk{)5UUq-*X!KO4&n;V%$_HPO?PRVF!4Au(Rme@^|S9 zJ;-}_T;L)rFaOi8?{+qQp)5GVCo*j^va|I?wr*_#cdFQP(m1;CwBE}3-P_zshZ zd~3LV?J=KMd}L)+Wc$rGlKC!{AWwQP^_1-aEXfOmdjnB{lW4(Frz&Z#sp00m$Rj>#0BM%nsu{oaq(*3ikSetIYs0duY zzj}`_k7fNwW99E$IwaXi=GnxD-9*k29cE|B)-|8l{hsKsr3g zv9c}dJY^+)d5OJ@*wVnhOl-ipNpX&t<#s*!GzW9-driC{7tk@w| z-xC5xsm0&2+RyQFoR_R+XSsE%V%4I7ssjKO)s1vJy4tf>QD4TPl zI|{MR-9EEr5*=fdVoC2l0zNmgU5pS8FVgXCQn9goqW5;eQSD*&J(d1#UU|3k5L_$N zwkIBrw{pL#wSHe1cWusHeA(-OM#~|#-uH3*9e2fij{iTDy#;VwyRtSKGh<@LnBth3 znIUFoW@hG?Vu(3rX2%R;W@ct)#+d1ivvbbg_uTW}s#i5rYow*6^{Kn1>29^8RjA0l zu^US(=a68XdDw@GC5Nzs8^WJ6Q3RM#6+4U-u{ap*!!o>oT>^6mXO<0KH_*hYezS8Du0 zR|-ivB~UF7XYaVjGIaBU^M-|Pq)Dqg$w@WPGwCYxNwXcFk-t55XuvMt)l&p z6fE4=bKkE!@j2X15YuGLTyoOg;DjHM>2ppH2d1jN#XAcqNmy`3FJkqixhU;P(_a=5 z*UTO@ovWGqjl#;P&NGUV&ZZnvn|o##H$OpV(UA&-1l_J?xJ?WnJ86>~A*9^p4lgX) znir|$!YoTX`|ZK%x0?rAqkCsve%s#6p*lK9_;_Mw(kR>+J|6n@pf1m(!=6A5%XSU- z;8!Ucf%3B;++67$hCglKD^z@YnfY2XzH?9eM+Li=S4?OIrAqK;2srIS{<7rKE*EV=*JJq^N7QX60z~m!|aX$*VI2u@K*(Pq&fhd3^zMT zlrQ(a2avfDkyF&`4j4V0c_C&u8N`i8Q7_1#1#?#sq>gYbNv8*JYs>Yj>-kVC%dDat zT%Y%h4w>8(A4iFemN;ws>&nU*eI!2OkW1a6ZJfP2wB8G1182u|4FtzHN2dsG7i>FZ zV0IvET6EX!l?&5;jqah?L%uw@jf*~TKf}rHW@P<0!{?t76Tmb49GvtWfGP-3)uRCh z;Q{qJg?}I8_xAMLVxYG`k3Z9nf9ZP6K;7(Ldfl6z2Xr&iu^0fqZ~7ikhkMi2Sb^)i z|E2cH8=f9r&Ay4;^Kzx74df1fhQ4pdM7p5FXVYz+SoZ2yjx`=f`x>5y+aAP@@+ z5W{agKzwg`Z@MPa8^*W$K-_OS=Wne4!1r7K{8Ro7KP%%u@cnC6;IHN3O z$pUJyUx0JLz|^1l$iEr^bOQdf5q~^?zY+Ye2K{yOp9uaPg8YYd|G!xC55oLYNq-ZY zKl04--`VmT^FQhMCPBdV{b~4H%+sHS|KZ4=hX3K$?}pR=4pjs`^oRKWY&iSBVv7He z9%z{TCnx{2KfTH0fA*obG*B+zWR@A&+5TMbP|LiXUdJaZ!zEfZTpQ?bg+<#0E zzd2q15d-&Eup|)w@20+mw*iexVCsKHU9taOIrE40zYD#!`)^a`&4>Y}6>XHPfq{uY z!-pL>E%Td}f4g#lfrVy-FJZ^_&0t*VKGHXMM~ATHv(-dDWHR0 zO8lQtf!k{QZ>WDH@r_x3K?N?P{6C@o5#;|0^?%FiKfC_lP-R5b)+z`Z;App$oRhj{*jXZ3h*z1 z_y!eiswBHntHm19}D6fo`_n&HJ4#t7l>)?PR47bT$Ah{M*Cu zHhc@aI*KYdzPW1NQlh|^A$Fi^>G!juK%dm_iAx|`M45oU{|sZIZzp5{eNcZDdh_M| z-ihjsL4VQtcU2;9p2y$2OaUJk`5hy5ro!7hWq}SlV99^|9RMAQz|@}}Cd>Mk z_*a(#mi=$X4bzHS0UHR6Vg7vu1~UJ?iUXqd2QZUlcc1M;5X7F zg)5-X?Z58x@qQ%mad_SU-m{}nEX})`D;Trc>3n^;^=W?k&ihgh(0L-(c|DSuHKul_ z*Pv|kc`N|S+nzR>c~)L-0g^zKi{OKe@#gUi>>1j!BNSJR&5w04Hd;| z`OrMVZa1)8mTW&Uk*gCgNl|R1$^021Uo(OwE?ARV$y_b&*h-?f;UX_IRMAZ{o?Vij9HkCG`uWPYTi*Ih>E=Xop&Yf!=g%q>#roYWs`nIem+SJ? zF0FY24BHf`u#z7|z^KJXh*YA5)FiD8yYrLoD46X&#z{Q#mL~s#m4WPgf=U^-MP1tSfWC~1zDb9v^WgO!Rq{d~nkZYv_L72|N-pZD7+>BURa7`xJ(J|17;QGQaQh$$WRx zQ{+ji_@$~^sx*K-DD*bbl0IN%>JHd5+#Xw~2^h#cU%SXW-K6fyVP2Ux+Q^>fzP=`( z>%4#Q$%lCjuFd!{iOQqyUd(=Uz3{okyrod_Q)`>}1YaDzE6VeVR7X=@LKy4{wjyZ@ zITG=tyPLv=_rjVQLbr0fLlZe*q2031NmXFiqdLXi!Wm#&Xv&`{#m-xkl2pzxUFh*M zMm>Fm}ykpJhcF~-I0q4(=7 z#drDg$%9pjA#xQt40SDPOA7TUDJ|;dr4gy}Y%5?Ld1RhwJ049b?lvy9z&Cb8PEF^{ zDmOvA_co)ctqX1*FwKs9Zkd7Xx2n~&9GEeztxUc_d+$pHsFd1_TFNx|&=mdLv!=m# zP2rTQ+>1FH-kOpreB(Vme@Qe)%Y(P%1%P;_eW|2*d|7C1lncZm#G%Ei(9E#;pd)V^ zTQ2*Mn+n*k#Pc0^E;aeq;$#5@eL0TETYL4zOy*=e*cqPM+J;-`ERdbU$qzcy#@WC}`6idu{W4{)x`?Nq=cK9)ujG4S~Pb z^8&D8?V3)oI)*gH`?CA^*jfYKw)65dW#i@Aw!B&}VFS$Wy*dNz9d$kaI)B+ZbN6Xm zyBB>PE#F()+VJvr<>lpiy?Y$xo#cEKJ9~0}DMdQtJul}zKf3_%yj_8xqk~(p*Tc6@ zTe>&!CNQ7pugq;fpUd|0g0;hvcO&9y{#tp|Z1FdOal7@m3jU-}Vpg6zpKXqKu6)jD zS8k94?9o--1{*k7;u74?0(>DdPnosOTIRwed$jgVV>!NA$w@-hmOD9?FHV5w1UAm@ zrK#hzL(6JSjngpWZf!^R7FtlxmY6ryBZnAKYdKDHyn{8(ELTcr(fn$UF5730J{`tr zZEc~^1jFQOvqAoI&^p16a;PCURYUDaenbdV2qb=vMwl_)dqTE`4&5d$>7fU!>AcvW zZmH%&7jg}ug~E&>5kjF=X$}rpomm_cr=0H8sh@A)hor{0u~U9 zYN-ii@3Rq$^q_c_Mp^=3i}ap(RDvdL%O*cEQ#Y-ErppCS$IWsyCs4RY3z%C`on{7x zapcZAGE-{VdgewY4=zfJ8L_lW)M1?8<}u8KeUR@O7|C_+*a#3X=U}a)hC#s+$ntBE z+>og{uc)vKf0#XFpJ@#(Ofh6`<95DMN|=ADYVYP4fqTs?Yi1$f1j7i^w6YxwZI z;sdTr-x5#fPg^#vZ8k?mcN(W-yw9`2w?kekM(0@0JVUcaWHT+dh99A&dCxyY9;;Z|N4 zPJdY48fnx&AKiz|Iase5X)wGnKjl3(Fl3~lhfVE0&($neozz+m)$EPxIC0O-vE@T- z>U8NC@b*16G{0Uy)%Mb#{Im?~Yzu005vdFA%JDpS9lE0-}bXcaYOtUuU! zLaJ*6@Rb%186Cz86Zfs|2G+_&!p&>~L~pc-wa5Ryq82XIJU)aiGM2m3KzhBIE4#fzPZwyWp{o(KPEcDe#C+;!*)Y3LQH;q8oHl( zPOy*gVvKske>7%xam>0e(I8(gtv=z9H;7GszWszjm!f)Q44&SfqErytoap|jI_e;Y z-x#m;rSg#}fp3ePKP`Lh>7CVK-!6`6nRs+prg6{nQ#1sN84^(r{g?C`|AD(_YOXq9 z{33vv)RS#raes5Rl`gI{Pj6f<1IZ(Dmw*NTiVgmIkfV^pms4mY5%$s>9B+QwEpM#b zq54;1oLeMx*7Lmfcj6JX;fQMu%I*oh(C~NANS0NvGiR&b7fVNHFQ*u<*K@C9J`I=G zw?3~Wua`$aT~x3MR;%stS>r+%MkADrtuV=B=(=gl#^(p)!J1CnQOVg(*m$WCvms-UTfW1(rDW`Xk#tZ$A&`?#&OSF zM6*2r*pizG*pk`9yQKpm@VcoLFiQ_bsJYQ|;xarDJku2TnUIgM*VmCOqGC`#>OO#I zigqK~ibCM-=H{o{(HY4|O34S+%%>`{S%lla(xuKfo`-0CNb2*^BWF9ET;1j|WvfHj z>Brkp^aw)5BZK{vC5?0i&9(kvuYsSgvnagj>0DKtD7}0q5A@7tBR+?$;2=kO6~)a&dWgd)QT^Pon)g+8c~}KiDYR}HKEAD_a$8dEBSBfH(kIBxA1n^LYd_H) zcIPIn1wfQUN=ZYIm|TZN>ES-GV>Hr+@$X10q-Z+vmDnP4!$QW)$}dTw@#7_Vz(`my z)ymlHXxH%2(>*!gwqK1_WzE!pD`=!)vgUBLtjyv-)|PU?^S1NrPD4hvk0a3eU4wk{ zk76-}pl4WI6mUl7l2|8XTW6IH!~?gYWwJiG*cX>Pct-RG!{6-$czrJG7)sRWf$;~! zSZkT#gzlWEqxccxFZf=(j5K@Qcg)tLF=w3)@aUP6E>dj=0Yfvn2pO~%>Ax2r;9qb6 z{Juwlka0nRP1nQ24>2g1$rS;Ha+gRa?k0|MSB zul<~RPW0E#b+qA&OC3+lV2BkbenrbCY?H8%wNm1$AzkEjrDO}C4xX6vEp(LWbSqH@ zS=(*J>=6Da`NUIXHhc4}I$m(I?G9FGT+*EzS5QO>w5}pBzs&2$&Nb7mW27#dU?8D_ z+NXZ`wNHgY`xP`nf!!7&f~RkYLZ@nS+_|W_ouAQhZ>}Yb`n3`Bw6CF*b?6|sc4QE0 zO2OF0xMI^=c|rHxP~?62o+4O8TI(Lnd*Rv$GvPqqPe$Bw{TN~)y%@K3id>`aFw@_e z{62+pHe!MKQ_LeUYw~mwOnmGV$Sj}It3kP#sb6yWQ=x^<1x=#-V-=A^1du`zygxbW z>{Q+1_vv_(7g46@2Z_a0Peyv%Drww7I^mGDeb470q1PO^yLHr%W^TOZ1l#UTUZ3WC zp7(fPHXg2#9>-*S9(ru9A2#M*5nitgWPIj}+G})rmg{zPij>%_O31*ui3Sysx4FZo z?S@Tq2_=jKxJlo&a+%gDz+nT@E`${4DijZ}Il zuGaW_IQ38WRuN52JrZx(Tl>*Fg0=4e3lGLZju@k7hJxp^Z4)O$A!pXQ$Ama-Z5rW} zGlip{UDEq+=`q?2!qdyIAzX1AVPCz4o53 zC68eGOc-T8dQPM%w$CREMtEeJ5dl+ro$&+71CS?VnhyDz$Dpc-j$j5LphTzO_ClE> zxkvhT;vIFt!ce%;omE91G~b9lOK&6^{A1edN=#_%oCI9DIp~c=f{I+tf9R!0=@oy< zmVzCSijKE>Dht!tp&rIdh0V{h#BITfzTD$;Uk3hdT8UR{QQGFulHN6UE-b=Sqk0>0$m57LL^bH)nGBfe>_?~#@Lag=S zBM#_!N-P&=MC>n6Xj)iXbfgXwn$`l9nX9lmXxd|;@T}yhX z*4Rs<2+ry}Mu_IF5v`u1yo{bb^|MkHa`JwLxyoWtj49xSTsW zAz^7m57qGFX=k6or0FN-G9FQpE4Iq5M)-SHgmuoFLo2WO^-zf0XLJ)I_PoHEw2IoQZ@?xFNwurL zcftL*7SC^C9Xt`8qo*l%a7k%U8HE}7TG%pj0mCp5!A2`L6X0oCGrA&btH zjseD%E(7k$wbIbBb>QaP$KCB48H~f~$dwBB&W+8JZV!43QbxlXWFXQKY!?qrmCkM2 z(2o?Am&Nt62ZQpHT>Vpz%=&cQdd=-B4(`byn5we>(0U;NMOh+nBgJ(OvPw;P+XXJX zORsu+kQH$*Cf6m!BAdxpbtGYlR;(-m$q-&&7Q%k1K1(15JNq0s4Zv z>LY8R@P%}tpEw9n?;0n_LI^<>kmo=ui6;Y`$u41{@?qa&Lj>n9SO;}>H7um>1+~VM zoP5XUh&q}=M5`&lD>l404TfcKws$m8Mqj#ISxD}w3r0falLqu56mKJq;*kvX3O#yc z>$$R-4x6#;&z^QjA;_3~?@gD3nBl>#PwLoErc3IYwNg)cWyRsjhP`qBdMMIj^{T-m zLmTDl*9}LaO?aU-C-H#BTPUACC(-t_FehPUTvHJniK zT~qdWH~F& zoWx!)8C{`B&JjMfb60J6V@Dno_*2I{uc&fDQud<@OC*ns);m=lrR};W%t~ z9T2#jlL#R$ak_rna&Zvx)CHqR8dAu@>miuDAV`Ukn2{bXhhpkqx1PaGmkH{U2@{r6 zcq(#krp52;=D;iCHLrUOe&z9;F!S#|Rm_23s0|X@FIzwkf{tG4kC$A9X8ILyvRZL# z@qY8_#iUw4-!i5OcIl`e+ z_wdul$ZIws!(BDw`EXA=Ba3jXjJNAvF`j>dXXwQU{3#Wt^BW^TRf>6y$S!?yR>SOa%c7C!zMGb*A&!Z;=rWoDN?8=#E|Ebpcr3Z zT`~8;5(1IRk+C1h-Hy(7m@EEuhKbwaDB7YSYZXe{sP%V-HlD|UDaUd;d}o|9yX|9J zS+X^<@AJ`Hus^rG6ulEiBZXfs(UCYUYmdaDJ~eBRtry*r-LVT2akrl7j8RS5TZM)Ly+Fm=_)v97y@t7_^>4)=b{1M0oW zBR2hnhrd`yNNB+r)cdh{L^6CDp)Xudi;1rD?sON>g!-E(GacQ;qhQ}e_upd&=u`P% zqBLQ&`4xp|WvOhiBl?4pbl0iv9>FH@Ds zR04+w_^jW-6h9J!i^FhBD(;iH_%I&!0>`v7({Zy`)5QbvcSGZQIV~oZqC-d++1$9! zcth-#-005W$W|92hpHziMH)9?*q)3SyKr3E>ldKkFbyR2BQD{xIAVBj%>_*a2Zv&AEHri_!b&M?$I2@U3FMQY z4L~?p&T0s03keg2t+t(n&e1?x*Q5{>}kwBd)=tOwK3?Yh@(xhd1{k210va|2DGQS()%Wq^pM!cY`XPq2PD1v7-K zWhg?qz)Su(5gaJIy)lT0y$G;~f*r7kCO8x<866~sbb35PuAJ}vuJevj7Zq28TY8>C zy~No8;i7So;S;-UNrG2_NVJ}=8(ligJf3q#Q!4fmLy<;Lw{?~RRi$dS?jH>ZSO=Tr zo+v#1i4p0Cyh>oOkG7QGD{6NBfR0RXK5f;7jCjkK?!7G~Gd-n5Zl5To=}S4DW)>_T{1<3o7)i$q}i;L5L`u?{Rc7!HL9sf|SlzYafy3p51T zO~{GxGUSSH8|6W0AQN+F4Pt}Xc@J+@@K-3|IC0H)E;~Gd5I!@ko#RgKUs(714_>BX zMn4Du$wUU)w@}stiPj)~B@Fpc{6Z448{1C#t=QYt?xazM3B5VuM=IT#}@3co5}Aw z%ngB%Z5ni=0f~KB(U|=C=G_H{u26$XW|m&vPh#WDtP6ffZ0s>CQ&Z01R_moNqHI^5eef^5eEVz$9;eCJa(=Dp|oy)GNsip`iqO3pcycXuZHoe7|p0 z$qpSQi4(ge0*r1pv_MrNaaokI@zBiVXyUR7EcR5e25fQ;UDiyFdKQ!%0#K}bgY@ub z5UE0xZR^qCch8yUXQDf5I@HhDP5oQ&2O8yUmR{-2ih+NeprIIIPw@fWIyJr$j`y#6 z)KxDpn+)F!L?-(@m0~atl*DI=4(oK?O%D>5nAS8X>aXgS^K(N*g90AD*irC4V#D*N z!)YraHWRs7`>JdUTx2d2(ilozyj#U-CSoq@dw^u-3qto+#LRWg5_m4tScLz!h@clu ze~Y45TME64(@7Lvc9Kcd>3{dNq8Lg~38^zZ82D8N>m0@J2kT59h5X9h1qQWWTV1_l*%EXk`(9gc4W8$PEP&lX~4Ayem zPIXPqsYy;+rOLeP7%dF>@UfPmI4$0B z*Q!1bs>64r@_JLxwpO86cR~@A_jsE(-t&4;lc`Nr5UsL*z4tl8LFh-=o*E?q8HwY%ho3Z}PIMp)53%-DtA*^0ROlAIIy zORq116t-`G8S!JI+(fP;3P5h)Qz3BBZLsbSfa}RpY`}B3CNiVpX9HvmQ$l*X@^K9Z z@WDLY({U&qBb{*!%EX~0bXM-8>G#?tnJ#JOy4e9Z->XLiS$2fo48;vpKR8C=g?tx) zq%Q3AWG;NuZ)t)0tpHHJO-U&`jU*`L0R!r{i6{s==hXAl8@3g##1YW_{PrEi-@{if z3C1^<2CB5zFF}7`)Hk>9(1QQS&>Jbx(Y_KN@EJ`m_i-EfRR8O_Gx=<<84T}~l_AFR zSzu>y!f%8inwR8d&8LLy%MI~NG3X9MFj0_`R zh{veqUDcK>D?6+8ZwgDtJeTz$9y9~Zi71l{BP2SdF#xbL>Q|^qt}2bUG8;(xHl2;l z2tAVrQ}_w`Un^eW^Kc1&x_~>$&dlfn@ZQ~zKnjn>$Fu>2H0cOd(K?G@4D z{-bqVABWsD(C^1a)l5v>GfoTz)>7MacDg1*VLRZ1IH^r<(=42q`lY;OG+ZK($&(vrgbJQ7BuX!T?tLCZ!+9hhw*P48EMTwDgY*^uSTZKC2 z{jS|8MAPA+Oj4Cxr$L$Ur_7$nerFG4gZ#Yxw3u-+1H;3He9!Rhq-HiJb)8wOa2@Na zLG^knKjPQ;6r^kLiB877EoUEBNd!`iEgw++jF6yAQV)NI?Gv6M^?Bc)MHb4+?^oO^ zVF$NHBIB<%3ksZNyKK1*8V2j9u+O)X{ zj>sI$@Pdyxuj)vjM`CK57Q4G&kdKv$prJk_NSLz)m;*#+FZUXvx!jiYY-yIvKIynG zcyrKNg3fURjmz1F@Thi>jGxSRwfJ6^UxzCa=bDMgEf!m%V{)aKy-i7;zy%ki!b>Cd zZ`=$Hp_FsN#B{6-)2VM@ENZ3^gTR(z3(z-ZGLYorVm6NV4Q!x%JhC?pIux<%ALhjt zxTwt7cs_nobza(T{`4|0_QZ}e0QUl))zq^v=S0ndvwS#ZY^RaZ=m%S9heS zSDwyeMRcB?)9~bO=2w*p?a_WJE(~bL>c&11_0ErGr**oYd8(>D0?qF>OTDGyaLSj0 zjcMYGFv*qj+{^Zu-9B2U1N5ASis>0z9p|TmRH{4^uuBm{jXx@Lhw&dn};(1bsVJ9h}6ZYFbBfIQ5;caeMxC-e=Ku^@AlF; zOmQ#^mf4oy*`W5m%f!}R?LD%NZ-jM-GIp9e3gMt#tmla8pLg-)DqA@in_Zo`Xdsx| zxV;={el;yl9`gL=fu^BtFYIM8u3psY2}vb!^a4|Jv(Yts2o0Qm(ClEI!oa;@g&v*gIi&$zL>Oq#ZY;P$>N52V=beFMlnEPF7;7%%(z z%{F3qvVOPK;9KAfp*Lk$LaFWHVo-7t?gjbc)1s;=Bwb2gOvQTjJc{qduee|)sHl=F zY!b8vK)bTuyg-mQZg9n&tB4NTmC)Wx44O}NUq3U>+aNWf6rd55FIyES*)|P`P^OIj zaY+_`*ny(ljeWqQE}W>*vBai)sB36Zgx*<%3W{tY+;Na!0VYGm5v8*x&%h+rGLN!m zDjrMBg@EHCGUeKlx56daJd{wc2QgVZmndLY za-0DpKAB3VwgX-9ZgMAAMndAnN^4~;7L}holy8z3mno{JIOvjU3W(I_mU7Sxf69JP zZO6{q^+cIH3KYH4Sy91yVO4&q*gY>RIHo4{hA{J`Y@kkiag>gr<~}}P{aO5hs%H;+ zN!VD4ZTJe%5vD&a$NuONWvooo3EB)TjhxmHV&3lFSUak~gbcRY>t2I|LDVfQf>Zm0 zT6G9!uJK&{ThTi(4YfOLNMU|u7Jn@1h|65d7F%isf-H{&i_+bEdnvS&{^#Bz$LZjV zL3kecpXd$OX0WmzY`jD_`~y;Y$xs6BVK)Y1Go}(sp2JJq=fcLZvn*0V>*y0HC`f6Q zZTMH)vpn$I^I7`-o%=<~)-~&0lu5=%0Z>@A+1j=>>L#CX<>+|_E$g`aN*ZkP5~(?l zK{gvbG?S?b$eCOut9QVXcdZtXTm1nT&Gz|m4O!oDg7^(c4?!yUEgGw&X-%P=$X#T9 zoG83A$sGuyK5GCap;x$xo8c+)U8LGKk>Mu1){dwn`|PT3e44fkzgLY_*z;v`*nPJ; ziFq_qW=jX*RAajGTaHNfA0r$kGt*s+8))t)Zv*s}!MBu2iMtA`KgtE1c zQR+Bg@Wc@*ce!bn4tCT)d#?M!{;oy9Z-PrcSE<|S1FcMAWn$JSq^iI2`J^#UrTVLl!{7q< z;=N*}@Fb*q6)EqmJk<9SGz}-=ZOGb~o(-$7P!Z~8dPMI&GX@yw4OWn(F1R**!eG$t zBD((~FdM2Y8^-=cCE@VB#Ux|7pQ7b~`z~+J#^4!?g1U&D20{k=jA@-Yk3+c;Sz5LI z-7U98u=7nbek-Q(yDS@zQrn=4A0PC7>Rq7JSN-_x=i$Nr{tkDRTa?hl@56Q-hCF|# z-T<1T9_^_A;IZ(se3`SxDFpK}Tvuk92ME>&PAcXAm}B9C%D5J11avgzhK`#XceeR` zZB{4?M9O2`TMugdb3NJxq^Pg8&mgHSFl_aXEO1aAn{}JID+|83U$}7M1JW%d4bZ4C zhSOWd__wM>IqLCT6reW7T91aoO?;ep!|P)?V}}+l)5J~B;KwY{08u8|OCBSPk7?8g zNuTysYt`Fs!`+=5OlG;8HmV(75}>Xu!k!oP+v?`asc-)8pS=V}5xUXa!eLrKo+_t@L?f884_hJ8d zp;8$e1ggQj$6e6e7SDog9okYR2YcRe(Zv+?ytd}6$m+8vM z5B7oIp10dc-utp+QCYo?V83{(f{1p14SEsJN15>nEmeZ&$56D9NUQo4PagP@P$ z$3`$_$`@geqxsK0d}f- z2$MLCo&P##fNardp+c9&OdDxxeEZo^%`njtF!S&gclXRj>DqJTS9#J4^HiJh!`zqG zu??73v@us7Lm0=aXUK`@q zGrW^(!Yp+{wXwG)4ae&+6|?cK-1+pW8Lp+f&A&TO9;(O*>n7BXgFm0XRDbkcRls7f zX&bh`jZ@~9Aw+^GNJNS2sVX>TN!~@pR;RpQc-+am+)eRrz^|mdT9a(yYTQbrpwP#q zSaYRsWR#=3NvPQnA!{<|=`Uo*phBDwL0ExzEO{{2mQ(fvtx}tkNoy~Pd2UAl_jeY5y~fJP!p3?{MZpP!pAQ0rfln<>U;0i-*y z_#P_+Q)Ybb;Me7~I{8F{;IxbzFSIgS z20yF5X3&7|A0hRGauEDEA@PMgGuMfk?t2?g1Nmsd-9L`Q`L!~Fdd(T7dAg~o*y5_; ztRy(`GB;d*SD(E;01a&JT|q|6H}=R9_h5abHZr9SgOmUcq@YHMxd{RBTltOiJ1(E& z$y}q*;S@y7wew)`sa4?0NI9$W^tyQr3Fl*N^)*)bVJARVZo=!$#>_*xqBF3fw&%`b zq~?&d#;Fh?gnAGjE%}NRn|yy@shJ6xda&wSVX;0_m&Xkds@pe;8Azf&y`byt`i(~} zka|K24eZAHH0J7en`7Ls?jN=iTA1+9jL^xlxf_~Z>%;aMiXQ4mVnzG0(f3)X7&1SJ zRyBY?6s>pT%0ax3`SH%q6!Aqwm`H8@@Jh}B*3?qN{ zeP51<%dH2$+>31~(%mK288@FWS^_+{jHJbRhCztoMzD(6rrbyW>h}Ni;U2}J0pqw?cMpP-cK#j^ly=tWTcV@U zKA_o-0;u(U#Q&U9s)L4xZ%@;jZo*br#WwqMDH21h1Bb*Gywe|vbd!HlA(s3>eal0Jj zh;b9TW{F04>n;z@zIhQSLosV6WTZk$F~k|`q2Xh593f`=^RUMEGYV zSl->MG?zlI?XEZ!u?XDNn-={_R!eFYAc~di;x%++IOCuq6H=yLRasl(*eI*@3yEL$ z1<>tFqLM~O{h{|1YxftqlEy>B+}GI(V`1=Kvi0egOfyG}JwPr$o;D(A4MU*CMhK8V ztJsA3R`*d854tYG0DJ5^f&g4^q|3p4A#J7{BJDN10+g7YrKT#e zdjE6i00{upUQ(Aq0DxqVcWBO;$YuMWH&6QWdr>A0UHE>f87}QFpTekl8E_Wr=JMl` zFx*x_u;QY|1Wa>Pau@8V?2c-t*F(VaL&@^_O>+$M%iyF2t+8EvtvbZ5s}BT7HNHA8 zp_?0$Ktu&0LbEQh5@rVkQdB*bb|z4^+xCDP zZZ8(!Ku}GqECs-#^F~rlnk>bM1|PE5OwT~~C!}^&2rrs@$PO6!g$?J2PRF7pksRs6 zhzDlLIGTl#f%E ze#Pojw=n$0^3aK;MYW-yW0!hR;*&WDiY){=k}u~H@oDmWlGiZAm$R8T;-i1G1wAD_ zd*lqKtP{CKu!}{`z57hNu}gymHB)8&-ii?5NE8YO-z!?H0oR%kt{ov6 zOZya3BxNBvL1_^-{{%v-8OH)-`lkSb>CTO5Ql+#^>8kAB-qSt^ft(CuRXfz}ZARK~ z;K*)PmA-;bG$KQme6^npvuE$ru1~`(eShTE_HyAu7J*_Ny0#S(enJThg+|M(sfVpf z^1gyrEWfp)M$&!&3H&Mxk%Q@%U;L3&HaINbiMa$b7L|$fSY%oZkK&ty0=ib~#tUv% zoLOmA!7_?DQt}yKk0;v;tP^DUClUJ6dkt!rXK-=hJ+R{!7-p#EbFpp+2WBiAF@#S! zyYRbNM)El)W4|1mH5Q_=N09wuf}*JWrIw)HTTu`?oUcX|oyCtNpX!D~CD323Kc2%A z>Z{p*lLEJn-hKVzI6}-?Y(&35r`df(-J;~ZThzdK3cAqH zzCil2%2XNVdv`JADJ#`%ya4Vt8xGhLge-DPSMsZ~jCV&Zx;JsB-4#eB`a z?)1|jNc&OqP)9;hCout>h`gHG@rjjR$X~u%DMyCdvTOcC~kj=4&~`r*tKFc zUqIEnfzexaE*YFBRuQY3+H@|FfQBiFA~{TzVVKb?k|Y!lwyv4*sO}~~Hrz8h{)S68 zUo05aLNn6Kf>0vkNAjrZyy#w@C`duF@W|l2PE*5AIMAx*FCaPVEY%NJhdsd&RI3&3 z8fY48WsVMwih&r^XAEPYtM@rI8E2$|OynBrvMxeguv)|dus6H2tq|)%_tMaM*R@3J z{mA=zt@C>S-KXCBx$EU>DgzjCEMG?8{k(SE_WJ7piBswu*2eXHM6up!;*p>E032nR z9@PPfd0f)7d@VSus*f{OcQxr+);Dq?%abNP_GkX-YxCBnk5^YBp`s)Ozo;V8l0Su@ zZoiZyIF@|0MqGm9518aWP##STLQdRY$Z5=1M-jU{K(KRXpn|YCD>9r0A%nB1cEYQm zWCcIoaLt)tEkJsl5_s%?!GcovnwMnz#>Y4!!q$8NNAF#`_zlvm{F|!oN3^3PHnivX zA>!9u4lCVw+l`oT%gH9TcoZ$-PxNz^4>7bvQ*p8Ve!j+bkE|KX>-uU zPQHhFzw8lyQ+2Irxn_zRK!Y5HG{{^STMmPToa$RW zn$QQ~9TAB}tX?CJZgsT}Q~I!?TcuFUPLkK@STRc{cg)U@$6pgU z_Ms3)OQ2XcRS-K<5%EYkxYwf4dNC`ks)=>icQyEix;EaGXrP051BO9mp$;I$*4&XZ zeGcJD#q}+FP&0Hb88&kK^hVX(vB?TD$Mvw6ihYcmtUlVQbJczLG;ZW%m#sA`KIB+w zcn3Gy*p>)!vhlO2$A9_6dsZvTuHAnAx^?iqH*x*G4KK$rP3h9dXet|sIKAan*cW14 zcjC0vQqw4dvC&3LGp1)&@~Abd9ufgBo^YpiB?{|C%pnzGeFW%5!qH5#_b{BGh$%Xb9Ud!l!(s@D(Qx3wrxIFE8WJzw zQqv71*O%>fFxtWUJyWaS70vlCF(r}ujS&jr(%~j!+uTq0=rD}Ugr<%qu{HmTb_AT^^s zW{n_?$Oc1?+)QIrYmp$2f7-`CX$FGNCk zy6f|QLmMYqLS58Nco?po9Hjf&w*x7S$3LZ~=2t^@z|`FFF~P5vpS*RF%J%;w?HzzL z3ARPuY1_6jZQHhO+qP}nwykN~wx)a9wr|fq`{BF8|k7Lh-kj4-^rTf~-HIU<9I zerjoCl~Ci!)aHYwkgHZsmQ>~9s9LHK2KPCMv(@A3&*I24vSm8QNMG&>6}Ild1wcoNTixj2h)*HgLSdFqL%Aas6e6k}z}~rXRv6YJU*_*h z{DHD1MMmx2(8Q_FQ7u~F(clF4@l3aQjizdq@)7rv0{EHHZQ@dgQPhNzr7Qd&GzS#yoHukVv6m zF*Fpc6@fmB%p75VbbOI31`+`pdVw83%^HG3KPl`^8IlV*k^%J~8yrTDU=3kX|deLPAx$3hFq^AM<`>{a6J zPby&2vD(H=g%sfi$+-~qJ0_ic@TmrfARj&SFql~FVLC`x7{N-6_Ct3D%iAAOUaLDX}F&p*A*Bk(Di zo)DQNtZqtOD7>9!5)|zX*DNZ)eM}W35szNAxW@Q!r(__c9<1GIjC|Uft%=ORH$-EK zG+LOWp=xNACO6D5bC#6HhHb0+MBIso%njFut(ul_T0QCu1|VX{yn%oRmysa6{8;0f zplnFiBaQc1ChN22dLGnQgS>R82^yP5LFoacDdZPGT!-oJk}#aFMTUL)n%f`eS$$OE z(8VuLMKmNIVxdgkR;;9I&<1=7u`F3#h;mh|eKcL=a_Sce3@>uv&`+g9Zaug4+I&~l zwbJy`;#PGs&QNbP3GSE-4RL8tL}{x6ewee#ZTL5b#<3RU#p>yLr8x z^)nel{TIm!<3qq;PtXpMQ&F}1rf!VVO~-a<#|p?Eqvs7g2eJxA{+T6ng9zJngmHbr zNOJU@)Wv`$NR!&efW*n@ZdF-8)#E(ns1xm6!2N&$&GDGdBZ&9<#6X_AK7qTEQ!QH{ z=~eQ{yn7F|eyVd)-MTYYL-f543aw$`2B1dLt; zcs*3(t{ir&l{|{~IPSmt+&Q}vi0{z#u$T33DdpVLJ8Hb=l z6kqh`;YAXZ=dKzz6%c-o>VL5M@M_I9RI3quQaYoX3*hNhl!__~djgKHi)kLGPm7mx zRO0)SlPN1cN)dL5#HAS05P_5&QLH0$V5vx%V}PVBf`iCrx1;<4*xkmDre0H4 zRjjTa8(odkGq?RAj2&eT};yhC(9cjdvLpInPSI@W#+ZFnwo(PhcF9N_>R z3!}lXo24Yl7{#-1pt!y6vhthdF@*XT#}R5O^j4#zjXe^;^J4X?)h;I; z9t9@J;w|cVicwzmN(x2SNG`v&%YuBp3+<&u8z(l3ODt8zqe=f3G)rf}m{pI+ZwNy6n** zK*U*OKrjfGZN5DlvWq(`YzOmo{Al4-AajCSwg$TeKI7;l!z#R|{4Mw_L-ga)aI2ex z@{lyg&;>M-f*8Sh+E&RG@F=beLUF?FJ58uU(6z!JI)i(5Za$dUIjS)T*)6g4*)~Fr zT--dMULh{fA9e%+{raoJF$1eWp`xG&u(55h7C7#>PMn0UT$_-4Zefgd62-z?*leny zPj0<}Ns+CJlUVrsMw2xnfcP3rM|EjWAYp9Oh1;h%@L&qgm3KA(`+TP$8;y8;*ii!j zn67--RyV@U9Er7wx6r_uv88$5H|AoAl$rvvv~V+w_R= zZ4$N-?Wd>+4N@=fA7yPE|7!N|*c*bP)p5J&j|@ZGaRy=T=?ZADWatN3R_Bk-^#mEp zB10V);KwMT1`Pl&@!1q)n3l?SgToISXjC|pmcW1908Oy8FZSi@)zHIzf+5~WKil;X z?A4e&z-5WUAK=)40XTylg8>K6dTjK~C~L)oDr?>NzIeR_ySFC%JUrO|9v(++%Z{)) z;#$|x@ogR7LESak0B%#R0r$rSIl|WEV1o@a(Vn?wI{vd1`uRMg%ggum zoQ$_@O^D{X+xVehgEGKfQat0^i+6+9Qwctnl!)>}r zJ+q$zv&IC$gL_8$^l6uLD45%$AdTdWiF#dQyoG-E!V~a$xbj_kN^(D?areuY`3uLp zWn$kw{ax{l`C|>rU@?O{#Tt2fBt;kr)KZaH`x%SeYU9c{W9hUxlZXI>5flR)tOCOe z_tDIBfHY&)Zpq*TWh_AtJwnYO&BC0838cS3TDbw7Nwe*&Q0iJxz&X)JKs+U@gDJ@5cz5|foB9}P1Bu|QY{d2rpWTg26&=O7M2d0rxFd?-K=sp z72>y7xY~6I^j&-R<;nc2p=_*8%1O$n9QsgC_crE7nZSvDW3Ig2 zWLa#^DcG58{&o=mo~&|)rkP9-{=8lBN&!RT`K0nSmc}M$)L+Xp=%ClvsDm)&msEHc zOQiS4=h<65;#MdHIfr(}IIF0Er83(X1$V#G zZnhC0p3>=@)SpNn*~*%cCLz8U%I5mbwE5|vUo(mJy~sao}$Wh$w*)rWb;K64m7iCIn zwlGH>!Q0Bk+PC|Uj~%I(fWu;fXu73JqrIXBQSiGN>XBXxdFek5^a*PFH#3P9sip_U zL7x7=lP4u4>7dwo>YDpf;F2hnBLtTNwGn`=VNXbPC3!i$Z&*1YM_+AR#kE6t_)k`X zpSqGRndO9mbqK1Z7NirA&+@8$3|`i1S-=wUW8`+D-^VyIDGREA8`Bx~K{ zfugYN-d!9K;yJGj`x}K7EQ^TSNsw`3o+mie>&XzA@|z)qb(bxio2EaB3_(uYv2q5r zPmp`HfF9gtEKE^gBnkY+DVOK67G^L~M6=3Syckj)1E{5a*&#^9hyxoa)T0o8zAr+X z!GQTk0lva?TV7?rVxZOKg_$2DK}Njud1hK6U%YR>~#$@V)T8u0R1j81K_&g545 z2K;Z2%;f2_?(k3w^y{s9M&5|#YIZ+WoV2Km?glPv?7?L`s5(%O))+zwicb9(Ng_RB zg(nB5yl(U#L@ZG}F%H%iJHQBFe6gI7C&aClFs2HV;t3ame;RqxHwT=SB|>641bD`F zEHZ#yug+|J`FH3DTT&3FNMI^?fh{jnn>WcbG_0Zxo%@rNKmAk)m2MVkZUvx?{JNpE zo+QQN?e0(_8Q4UC7#4fy?o@&a<3x;x48Y|-8wJ+nfRxBff-!>$f32M-K`2me!Hp5G z%9-DJU4wc%Bj=op_H1@Z&yDvFbeSzu<2elAtKz|UU7c9+ECxM4Mr|CZe5z>SY;J=f zJ_nhaiH{7u?{jxnI}Gwc>4n@+Ha(K}o-CK*kZvrJ`9tTo;=JBsk&Sl}Kus)WBq`$r zMi9_=7yIp|l(FCjo-*c#C{*IBmC`uS8%9+Qn7SG%TS(QAF@Dw9r93DZ0U$3}|8Kc% zLkrxs!)3oxiAF-uJO}gBD@R-M=^vRbD9}BDKRO)R_Ygc|S0kppU+Z!!zp^3hm_tAu zme#)Lsi5Hrs@G0izP*Vv7Vt4%H>Ytx{6No?TrBQ8wegAM+}|IAERJJ)Vo&DTZH??l z4G{Y|%Rqs(;o3$7R0E+TcLUA6ui^5A_jd9jdaOC3!Xha!TlMsd)Es2asC^F#JQa5o zG@3ywFFJt1%t}jC3B!2kJfG$aGFXsHUxXVX7oL83FFc45kpk^}B(z8Zg&o>Kax-pl z`6WmLEeXMYCfg)4_T&EZx5g+tH(fu(DY>6t*>(cb)K4RPOk6NJka*GO3lNtNRv!N$ zJc5sUd1Aw-s9S(%Y9KjK%*v%hAnC?ppC8H^f&xK+_{x=kaAgM7#xa-wAxlO8j?k0$ z!gPKzx^Y6bysJ;#b}oV|BfZUlEOd6lmCsfIB$gz!W@8$s4qs8Z)8H~huQsM$2&Q}nWYp+E zG^!%y!gRL*rIL`^#TqprT#s)FU_EW8KUGfS^R7_$7^F zAHQ3a&)o8^Jy+)yVCL=}6Mvy&#}&-)`R)wcgJWrrQdXHP_!M#HLlPV%OI5CI<6LXd zwiGv9|1IlIdC`dFz1Ede{!ZyZ_=9{haScsKjflH@j_ZQkjMBVa?-I9#M&;)IK2?t} zUn@&x3JH!$wq2Jhs`Hs1+dx~+4t4@{LdF;3{l4MCD^42Q(m5`!DJy!>1@iCy&&{Q~@Lr;|mUJFHLd(!GQh%#Y4?F6{A22|m># zHih0g6j@C=zaV><7U73dnw$d;38b7d$6S$8UC@;#Jj+kzkHQ(9R~bmf!hPCwSa+>} z$^TX#)>whnC$S4|32_?8ou6HmLPM^W)84o2PC#e9;JfDQ>%fV7>Bw2KTCGHvW))$- z{$omUXQY+@E{tqU_v7}SH%HRw$AQe86e9^qiP4X@~E7vY?tVSJauYcx=b; zoeg@#F7AEO*qZ<)y&*9Ap_*7Hq^bmSe&POKSloeGEaRZzpj@yGJEJ&GMuhKkq-&1; zGO@O*oy?}#TbqoMa{G)RAl5sM%5@Rkr66w*jkH`~00g^dCjkizVqd7@f zv@Me3QO1-3+KG;OE%wQz$w6NX^D)gCvq{X7JBl|sPRveGkj7y4yja(87!ltzC1R}y zo({O9tw-Dox;1Ruy{6o=V&{c8G(YQ-F|ANuclv#$F?7`(-loy50|gAYZ4Mw}@E*1^=-kj$ji;PJ$JToVm62QLl(Md+f)pp6ADqdyHLLc_H?@qA7iEpE<($zt@>lcObm8HsL zOzMn!P|nG7La7#ObQn31L+=E;2L0q4rgPA1z4E7%-cKjLFJIa07E2k8J&20;83-b56@R-lHxXFqeBF5O`i7Ai^&>{q=h?QqdhC9T#@jJDC@}7Y#A^&$)b^9 zDuo7r9_nN0prb%v(|gcSJ8MvllhAqtd`;6RP8dYrBPF_@us>>&h)(%&qX)_u8RD&a$FrZoMX5=}&!~}EAM2&Z! zA7_50D=K5=*0Uk?E&>S2B(CYI6b1KJ@jq+BtaYccY!9sP*^4<3vxE+KS<*4h%ZiUM z1|BipE$?INZh^k!R^OhI#dEQD)((+fD2HUu;oda1aTc??*=w9$=Hr9tQGx7_r$E^9A&+@!yaiqp)} z*a=s+^itKsC=_m|{b40$W`4C2gJm@f%*y?k4U2|e*rz?raZt|0dwO`vy(nOX5H-Fh znUCVEY299h(i(mq&SxDRagd)vnhItkyQ+Gc5lVA+g0eCvvZR12yY%iI!49+5+)_%A z+^*(|W#{F6u$V?`DP_jS)0*s$Xb-MG`N>p|lsz6Q zDT^HU*Iwe!x~grWuzRRhg@3wGuVMR!#Fx>dZm#`#8@^Lq9KsEXRA`n}2Md{8ai-(s zfgyHlnYpK0GS`^G0}Q~(X@3mV@wk%g2WC4@kVEemyn;Bfj_(9q35vL1$YVYovYlzC zT}N-b90aj6?_mIAehNXxZ>u_w?s`jetGnoXTi^Wz!Hm?%B;Ssp6#I>sE@&ANR>>*s zJu7>~%ZCEH-{z4_fEz`2c~t2lu9-ppdV0rNedb7c;&VAoe}vYV)B3&mY2@unkd?IATF$Wm6mAV~>jp|AU zH;pG9_Vb)F>iOPWK79on3_UPIVH94c`Bv?p_=QaX0sAA{Ez{>V-|UJxa8N(x?ZU9n=C#{RrW9v?ca^o-{wIY2LY` z)U*w@BA}oddyrH-L80V>_j)m-hro>!0{R&_ic`nfYG5;uzcB_?R;6VH17uOTP@h?$ zD=?d;tFQT9v$IvwvP$H5#_s~!tjOojX|Cd`s=o+tmwJ=X(Tk&tKBr{h?T2+_W>+cj z>a%dpocW6pfS&^OMr&U)BRKGF3`MzJLgP8jSf@MQ^7V)JiTj+rsm*ru(6ydJ7 zc3$%~R6;Og#jCKl{LU3fnRuZw8*uNATL|s}gcrYvm*egu_M4l$5$rlK0Bz~DHKe;S z2}L!o5wrf`0s5V4!s{U%WEktG93pQtMb_H8KL`|KaB3Qk0z@A#icmRoXU2nkWCDUG z={b7w9-qA$a6p+zD1m8r6}@u^YG2Ov!4)1P%pATosjWwQl$AqCE9DwD?8 z?g#li;*ezK#s{$R&s93MpRXs)c6J+nxOWsCO7S{{>r=SvNNlTnR7OcGR1QEhDW|xv zR!i_!lGH63$sE5iDTvN$>Oy zGl%r^%fO$?%V6@`s{7~UTld9L$PzF{s_-@H>9K>O7`nnhdzL$cxd;^qo=zaaFYkG} z=I6|SWOCWM&Rej`sbQ=~4fMK~=HKtqIo_7`M7~R}I1zX@@E; zj+KXx>cOTHAzQK)mTx^Vi+^uwGY~%GX5o{mXN2ORr(1ok!7EpRv~I+eYN*o)=)tx8 z5uyAOA7kUpFSqplj5|a6fJ$x1rlW4A{h=M9N#K zbe;Ju1z>45+p1#wC*sBRisqQa5-L*_Cdyx2WJG3nYS7N8YS@EK!S!^%9*_yRq#KLM ziB$Hyw6bMY40l$bl9?=7ia7fay0xwYh_{4s9+8cwIF-6=byrZNX@n(todg#1&xjHy&M^fh__zX`izlc;nk{39bvf7)UnUV^_BN#C$#n*dEahBZY zJT2R+ZX(bEs@>+e#v=Bg0Z}?P-df}vwvA0LEv?QlR>%_C9xr^YY&TP> zJ7BGN`wo2oq+I(5xd~3_Utu?l@a~#TRFEGD7X>;`x{Ry6o| z0f-#{srt^B{UP&zt-v}bA0YtOW*0!fL+zi}M)C~k8c|y^+;>QM6e4t%=IdFU2VDuI z(Hu44^ST#&;GW_Fto2AIiInjs!fmzz-gA$0yhiapIl%U_iMEFBsr;<~xP_B3iBDET z5U3-=j$>ogD=?>srBNHi@(ZQo{> z+K7E&E4FexrHyq3u2R?Y?vF@ji7bxs8PEEbw&Gi}ut38)N^x-q+|nrBtye^T*zfOB z!Y8qGOEoGxHAoEZ@E?`ki(IgB5H!H%(q+Hqa=f^ zn9nkDjbU&v)0Fg8c!70ALHDC-MnFM!;(APoD*Hs}y;#vrLl)e)_ja|TUW01e!hyFu zsqLeXtMd=07^#y)>`w6Yp{^BnodBZ8{i`PS)cRgO#YX265rOy51$agM4^!kMQRv^- ze#Y7(B^2EhuuVh^dl;t2j>Re^Lhh98TFvi{j!F2Ny=o0hCvx}(sAe76pFU@920@R2 zzo#(xZ1&>A{(yOln83H;)e7HF77PN{;_}mHO%+NNc$jylMv>2FYH*#MgB=i!6wwkN zdn?~#LAh$(93%_QMBbzXP|>=nz(qh|0~OB+Egu-^=AXF zgFoYxLqi;}Xj-Aex6YVdY8g&2qtqS1wcy^71+F0D#3OhbK7A-;>T*igfVO**1sGC6 z=Yms=xmI|R%S9_)i4q)g_J z_;vhj6^AZvXe4zgzN$CL?`6XF>$pEG!W2!GqIN|Y#rJ$ts(t#>46~h2Y-70cx821v zZo;;Dv1E!jJR`fJ1BXd_pB#`HWJIe-q0~YLgJ3j21uoSyxt{{y1+;DAwDvyKqw*5ofSu{t>{Q_G@BP2B`)aelBpTp+)Vfu)adV4~~P(_>bR! zgViWD7Ft$S(a8F}2y9!TNfZE#Q6!l{f3|{Cxmw87Kv@5{S-1<^OxRQf7-=kL!`QG_ zF!2;saXb+?nOG!N*g0*=K&5bQu^azJDh@aa#+P+E_#~16drol{UW(XfrMp5g@KEI5 zvUx4~qfjL!enz3LEo^e6GrsaTq4(YvibFzr3`ClA`KAMt+i6Bv*z&8qY5y=sB_aQHls8Gnw-k zX42q>5izY~_UsUx!ZDGAiEX5;)Q3y1siLB45vZ5CrC%q~!SY+nVVn36b}w69kt)I0{)OEB0Xl zILswlaUm0zqywd4tot*mCc@D{{}FWo z_*7Cnl?5&Tvt-(=4^x};dC{BMq5>pwff*lUw=5i@NfuJ(LIJeoHG(@vVa}nLc!Jhb zc(Y;9dt-*%f9H3!ug^L(QJ*e`QZj4 zL;wQ%Nn`>unenH7|DX|QvuYfS@wffd7tz|PCeRyV@Ql@YsCYgw?u9j9nDbkoI00;6Sz4t&0+U`Qk+NeD^mgnczYm0UeBk)+8#K>k(BM8y5K! zSwc^h?VE8&7hy~kr!Gr)$k3E{u5Q|8#ZVts#cIDG^)c+Ofhs0p1wi25Fl2(pqmeGA zV2*$#5bXt_c~b%^27su88``0yBPv}xoIt6cP!!n2k=Zj(VUuAE^=SfPW~+y4Xhqm% zy20>>wRL3@)v4cws+g;37?+u$g;yd3SXWD}rg4LFw5_vEZ9# ze)SR2gdg&AN^4?eE7m}%6TfFcJe^?k0@HK{rRwvLe*4=k{}So zG~&$(eKC0u@1o}kk<99$q>%Qv-gK)!L?WJvY4D($gd*ZHujlyG}v5-$X}=5r;&%ggrH%|K)-7Q!WsFmre5z z0(+5lV^qRLgawBYdi=RY6a7g;q`1d8+YNh~x=@T68pmo@96f}(CDR+RvSERXsJ4>? zn^(|2W4e$L(<3Zi6N1h=WFzy5L3~l~m|Jvo24#f%&qBrdwMzJ+K9>b#PapH7eCDm+ z0BITPa?-ih5+$!(FUG`C5++hn878i2mnH>Kf)*oDk-uBa%5I%C z)m}Q!8mV=dPIb$?op;NamxcIQZ!_6fU&h6<-gi-zaWdeWMWr9Y8wm;bw4^e^Bj~Ox zO+bvc6*XVWS#7n2_hfB}{;J2(KC+1G(URmV5@w!;Clw)2$>y@iukUp(KL`m74<~&U zwfGYgPN!$sFffZ@gvseSvp9I2NwFrTcCs8N&1r2?G8741bs<982qmJ}h$rIJ!Zb|U zC_XH~s6Bjbekny>GByBXnJgIF9XbvvP0@?>3CQTv;9iVN&%4$|XyY(f$lR(RCnbA2 zj*b9%-iGZOr6D|#FE*DYu6)0J?y#zVi*{3A_)th+alj}DVAwK|nQ7+LwdP667qSC9 zCDKo5ZpgCN+@2mqOIXK%-YjJqRZ*fX;;SHNm~>J}ujKr?2J2<0>XaG1VYO-Un*~eN zw~!Mom{8>f5gQ39MJ8)9hdO=y`E&>Z!pv#{pE`BI?Q{?#*vxVw z@idl%17l2aKWuWfXXT^V`Of854LttfIJbtx?e@Y{GC>G@NKoq(#NITl{!^BOydM@@ zvB_ngS~)Sv^GPkj2XQYZ6XM9)Iec)Yq+w`Ae#sp3LgCpUvfQiEiF*yN9I& zy8npu+ZwGR4~dCLFo$obe}ED>Y%?vniAY#);#7QD(f8CZbx==Q>)Sv%YNlN^(dD*N zN?tzHOcvI>H)@`V9&Pe^A1$AwwOZpcn`NaS2mZHRqoW|6$TGfQMR zJu|hUk(KG5y9u<0X$`$Rh0*nkpp__Amb1O-^22}G_H2rsz9^&#glZ1{pas5(+!ruy zq;a%C4P0a#`jvfj!e<&af54DKEMLqpMjMtZ4-}H`G%@UZ3q}19++L2szvg!GIT;o& ziF||79)_sUs?pC>|8zE~?QKw+m^68?4wuGQ*|+sQmR2>ORSMaPuX zwc1BLg1~r$-$%T{f<#MzP=r4O8>qA=0J?M=X$m?=iiXoLzxM#!9)$j4K#4@(v_FnJ z9Z>`KB1Lut$fXy=DBn*z?WW)@_k&-~{M!Qpm-1Y=VicIO!sUgaOo9QG!?X`HtI@f~ zI29e$g(r|#qExSJy)k%Tu0`m3B#+ez>pRD9y4!VlvMu{ZHCTS8yGz3GhOZ49ZD!lvoADf?|HXs%E%48F9%q13&)1F;77vtVi68^u)38dqNBG@#~vk)KHAwO4%=QJK0awKeJcigDeX3eArl~D z<2Vm-TWf2Qwr3TJ5&4tMdls5LHinvBdPs_XW?c$mA?yNHanTA{(N~?(1Vxj_q$*t; zHNaR--C&ipF{oaiG?toxeuu_)2K)Byc#=p~k^YHcX5DL!Gdg7_1Ka}kTd6NfngK+msc{k%^}8NDv{>!;o5){T8^`BrInjE6mf z-i`7c-2=SAtBTg&T^d9Ih|#nyHs)|=S2Gtg*3m<#G7=oO)+p84reP)(k>dswy`5yk zOZ?E1PQx3{B_>!HX4>%x9=2N;Q~B-^&2j6ds1`Pi zQcfg|Qq8q8wM*3F^h+=j^+{i?L_*8NNh8Q@m!npj(s>LhFeDIvG?oi0uoQ9;TQH8A zxn$(tHr=RFXlGE*yIx;SKh5Tfko-=G$zll%>nz`P+#rpV%FJ6K!7`6aPOMO4BrK9! zKWxEy>Na{{a&p{}V(=N1>pJ*Fn@JU$l@4dNO??|w&;F-X{Q6juxQMAny2u>0=m(Rb z0&2lbiN?3H6rn=S6DoT11v`VEYe4%vYcYw3FUvl+EXu9r9nwqI6`wLvOQo0Czg1mg z%tpM|**`})N63s$C`XH0(h}oV!YMsRBOG7wvJ;O(kFoa!;3os}5MG|Hs2y(-;7{5S zXJw&~r4bd~AqtB1dX8nqYHo0Dm*AeEnmSE0orWl#KF>sW#s{Tm&5 z6`X!Apg2vFaU3Bj)8hY)vgNUut1o?5;-s8J7UqGfW$xrnji4pU$32}z8ly}>N&Sn^ zW{d)@U?h02h$=D6yF21P&4G}lE#FAfj;F*nNo-hfgAPE_E+C1-SK2<9FVS|Lf-|jQ z`0r36&}Lo2S+(|zEe+9Y@~uL_2S}hcNGf&sXkS)n)>LWNoM5a|yDC$wDpRRE0J;w} z+y91HsAH0yagv=|lF*v`!vx!xzd?{TFV$-z)yweA@0i}}f zR#X^SwDnzpt(Kd6h^!kT0=IyZEQ#@2jm_GQS18XJr572+4;hZMy%!m|4;k4H8J4xZ zD_vB7gOn#o&mZIVKgK;K>ri%o<9{S6ubFA@m}%E+5EIaLw{j#lojBK+AsTheQq=cL z#~Q?w9#(RDSSoxIYcq|d&x5MEcNhGMsg`DzJODp$LS#UopSat2_JBhB{02zu1t6O< zC6Qbw=ivpZ$##Lj`n=(KYHTra1A%79H)z%ux^kZdA$)L*0Px(vFF7-Kr>_3qi)b?>96Cba&feQDpUw_ph3+)738(STG;&dT+xki#jFyMp<4~|+ z!KW=hV?)LG_%h&Q%WM+F=W345n%WOj9}XUF3>W~cXx+C7-EE(z(69Fi-Op!UUyt|C ziPyW2nO|>#aIv(%zEoena(BL%E+oK3MF7#us(p#hDv=jO_#iZ2a*b5Xb?ye$U^7km~dpMo1QzQM^KXph5ByeynzXq3m5ymdZtmb*e$-EGeU03A9Z56xIZ5kCb^ ziGq#6$HY2tP9e(7oFDFa;OSUnORk-$a7JXCcf-E?3Wu~2bahXlR-25?B%r7^tm@i& z{=&{oSD9n36t}$7FGW^l*x^2jxH4)qIz|L0x;%`s4I&&sWUMVU7Fk2I2kVZVV7(CE zVvLwj9MXW6nDykES`;joL|jd1U;pvgi$NIS2?O9lT+D#T!wtAK(uHeQnY_b*;)fN1 z@iLORW^x``mAoj;7y*c{dCskrB1x%_%v7?A&|PjJUB?8kEV2Ac){CQlo%(jp@7rYA=Pe%xYbg804DEQGi4nw1(0$4O|Nvh>^%9A4AgmQX#Lfl(-E$IS;E zzd1apa5LL9wCoB4i44Gt)^+$9B3l>+ahW}c+mdi46^b22Wt&@qtgRl*hM%s7@Ti|6 z9K3NZij96L^vg?x@nfUXEEZ|A&|fCNoA?WW<{O{<%!JJq2tkP!Jtf&!>F!lIKwq>H zUTK`I!DPuvxfp2sQpd$H=)Lc-ie7&BGF{K!}dR^1HJqiVhQ;e~NU|CTu<|uC0 z!5^A2f&cZxSsrferL>f;CaJKc7<<)DVZ4X+(;_k7X-i`bJQKR4;4W{fxRXfagu-&+ z68FuFEG1BEvHph$6A_WYqf^jnVpD=n(ADuL=B-&(d)i{(VHpk`87n8IdohcE4(O3* zAbsq*@As7WBH+G^UB)yr`T!Layp~%>fZ=9GX8XTGc?uAC^ zJ5a`1TF^;E&T6LY4Zj75*)USHGyE(lPEvSUzej5P>t)5GFm?wFc>q@j3m>)vzMS2> z)X)#_uDwAeM?{7ySESo<1rXs2MhEFN>>|ggpHS1}+^{3eFKK*`V2?3z;qY#HrwLBgFeeEksWxSCsAbI70aH((E+b&t)i!+>10+BFhmHn>EI)1ue2YcX6V^$w>2Ya5?!%ebBlxZ8Pl=`6BR|9PQyPdBtt5nEx6gsb?2f@h01%{B1$hBq zZ{=pqw?ZvJ># ztV0X5NJVjx3z1sOu-Y!$B9_GEYu#spE+OTIUMSO0$E2mROs8|R;x?YJ zn%61!4-TgwCDQ4*mC$);6qd}IXiSE99ko<@Dxv^V#|xSY6gDp|TGDi)x0@GK&z3Z_ zB=On;_8`Q%UtwHw4|JQhJwZR^>}@k{dWLw;)ICDyVxnv7j4+@*7XxXv4cH?xi(wEw z$9{Rd2MS8!h8xN7#(*CoY*8uS02XARa{7AiD=fmt z_Dx7%u^c@{)ZVxr;NoW^lTBnv&>=p0`90vn_A*D~Giu1L{eJw8{xzB|XUltE&z^N5 zijitg{wa?u3TEDk406vLHnTMKO}5roW_+ zij@+e%UYg{6x->05oHwr##i_vSXH`0Nb-WZnNKq5K<;{(lk5 zzq!}{H}svJ;TzxmA3W>7fbefw$yV9M{JUY}zohK{XYBj;&HurDXZ{Q1{-6IsU0g&$ zL;0Jc?&xm)t(Liiqm!VS{x>`Pf8o5-(|r@e|2s(iAI1KamfyVZf7JM2Nb29s{oBy| zSCsl+GWh@0Qbj>gPF;vn?XPN1#twf8^$ODeRo(w>Na_Cv_5EKD=rjoWd1KYp8ntLc7?1 zWDP8g4W0g>mo(kqp8jcH(*Nl8-|UH2`Y*7a;h$C@&G=XN=LDB#`YZg?)}@*M3jdsc z(ky?4e@-sxzmR)|e-iJd+5QUuLA?K~#(&t-za-c*urqxt{ht8)AuZ|HO%}xNGu7iZ zPxK5Ul(BvK@WdYg4q^s+aFN{5!$_QK^*FwIcTZ2rC7MNxTS<$C698Flm6ehQzYXho z2DviAm`0meCZa;qi?sT8|4(6(650$cJ+p+LK75l}) zJ&4Lq4WfY7I_`S$h=f+zBj46qYZg3Sq1JJ6#K`{GsT zqOR$Edh@cLW~ag9aDOgvrb-)}@NPbRo$fr&^xSY@?|cP?>nD)j`?@Y0<*v0Ov+EBl zN*OQ=vW&9*k(UdZ|5*N*{}pQQk#$zt*Tx8oZt#b~d{zI6?e0?3@5}WG_p9Y2IyhR4 z)V<)m5N*ekK782HKq)MR;e2a|^Xhjv`Ax?*LmHY!&(eTwEtL|D86(KBni}jAE1R`kqc#;Tz zBNUk?P8sLdEoMfGnHg;{SiB^2Ums_*w^*^*%$M)B-?zI9arWlnMI+DLCIOK{wnC3kUd-C8qw?1TnJLccQXX~> zjqFWSffyZL9w6gc$R|H8(mRAg{R6Qp^2`ECwP(*&dRd7H@A_Ps?Qd}P0G~`r1Ftf!{fP2 zyFQ!~D-ttAQ|VWODKV2BGsh}{DqVNA*b_Si%1*p>PmBahFb&?Gn{qv0?_ha7U0ZwA z%d**6Ut1JI6XTYdF^re~@~YDTHPz7cULGNPYv3W=YrsqtxiyE*+b=94l8hNbj0t&u z?ocQBq_@vgq@+>?HkgidA?HwhA^;#c$Fxb}mY4xqIsz{u+QW_H_?1nU56=XMqAlbZ zk|qc&VsMNnOrjt3z8lbv0rLbMIT^9~0grd+||B@K9#1h-Iq z0j*CZoqRYfgBY!xo&olnf_mxa+l}2XL-%MbYFk4bR#W6AJOCkbt7GmhwjQ(s zk^hyk=@3g&#)ynR?=xiH7j>jmJA{uPBr#7$!>mQ?q{BYmr@$sA%1_F7-5qZ9gaK&$%9n0qFp9lPht5iCwi?EtI;W+6mxBKA2qDMCt zj6%>7MP*H`Bq(Xq0!;8<ApkW=I^6pSptMehXj0)tJo1Wqx#E3v{d|)j|yu?nkULVxviRx zEnqU3DiyT}CiIg68ZpK7!WzH|thG}tVfr{=c>Hk~7RpOiGgM~ofuuKSy9t{7?EOXS zkjj3PgKJ14jRSqACRA!J$?kyBILQ^{Kot6>Pr3rGwz^YuKthJ7%MPEif%(Yp90MDQwE=JGK7xDr2UuQF#D2G<&aFnQE2lLb5&zqyLSO z$GL&+(&lk&V1$f4PC=Sq>51yl{9SUkxsswYY+2D^vvlJng^>(QoHDh z#4%P_qbs}1U_~A2gmH&<98URs7i$ZyoHX17 zJ#Q`+YiLWWbq+5tRJDQyFaHqE(Vw*A;o%ipm>+8g&=-WN#jH(nR4hT085H?CjHnF< z*Lm?6mXZi75PYjf>%ETdn_R+geHSH}A^VhV_fKoe?~ zP0hE17Oj#c^DozXqOvqNGra`VpJOgex)s7-e^O$u62ltgg4KYoOKll;tj%uV8k%SypB>*!s8sN>kTl`^(fa;^32t#!ev zizDjTcz$_)q=J0(73)2*&rbfAV;_O^kfu)_V;hq1rRsJelccHb6f$*7)xniDe2i19 zQ1CttxgM@}@0qS82x>Sk#ALJ-9TJ>43yqe@1jH_x^f*akx|Rxedo%^@W0hlrOx4WS zIivfNs)&-u~AjvR81y4OVMiMMybFf=`MsoT!6*5Foks)TA*w1ViY2$~fKL6bWT znV-HLVDaTG2{vHWOsj!uR@Z}m(;s4_5kl|e8Xb`h<2qk*I^8a-hpu1fkHE6BwAJgY zlw*C+%5U}#UKMP@Yn;}K(ynd@O&b)qSFba29fXtLWC%D_Y{09}E61*zULZ2zfI`-F zbhdIG8%Q2oUg=Vs6SHKsiUOa+ZA^m8G#74|-{f?y$trc#&rRzID_uBrWOF-ug1jo( zH5j>8@||x*)6Z!D9F?i_60+6J&lqNGYDQv^k6w%jW{m1<MvH zfu*XBta34pB0YGi#N^y=HEMZ$N!oxzu+l<_?=g5JG0C{2S1DgC1rbMv2p=7wxQZoB zhVVOEoxAH$}NaQw`UB> zzjehP=OP^tY)xXBc7!+rPT(uMnc2XzgLEt&g~gL2`H443by8ue2sdybO27OfaP8G* z=*@2HKz0;+)7n`M0voL0`h@Zo*O+U-h4l0h0ih9JPfsfYj+}jzSI+( zmpGxpbXT4Wggf5IpIZa3dRCD^Zln1HK!v~f3t&+gU#SV=vgO-1!us%bH{Z*vs}&#% z0d0sTQWf*!gvMbwLsdw*0#w>%x~sReB|oTKjuP;kj8Yh;QxK2HRUkOR3649N4rN%l zl|2wG=nWL(4uPVfeK#{mgz!QE@ss0c#JyC{`Vk-? zVAj4Ek-TQq_kgn8Pxylc70I>vz+Kt+f|8V{Oa- zI!55im+^(Ryt`-*tYzgQ)PXnrtK<=*`my?WrPi_ISW~I%`m!ztLxT9P%3IIQSLf%g z&V;WL_dZPho%#Qy$oxj8e?V_`W)30mvEz?BC0{B05CO{bG9}x{t!?I=#`8eZJiwqjUE3$<$rMXZyo7>8@&JL zJpJ$3O~?FS`u|Iga}Cf46l)}O{=zd@PlFKQP11ImA^Pz3Y}0H+V( z#^z5$iQf&G|AJ_V-wi(=*&pde;x|z<|E&sr^!mTY^kR&(GUgu-2Jl-@Q3lvJ{t@eF zoo&pSeye34r~Mn(f5SiX-?;wgQvbqrg+F`#7Fz!7`FAzqzs&AT>>rB3zq0$dmb4>| zIDE%sWon+6eyN{#PK?^iN|~y0f60n46+}hfLv@XnbolH2{qg7c0J37jj4I^O0xIOA zZqHAAZ6=;qPM5c@d)p}(g|G6qEeV&ad%EJU5HB64*6%=_8()v~Ht(%XFHa3q(lK%H z?7J2%Uz2K6yWq138&4Nqb?-e~uRtx%?3N=cu7{1Z>7~1yF!A`%_7)_5BI;Q(nljlG zIIGNrrm*j9nb-n-4#xDOY;uls0-mC*b{$_!O0?_T8*f#1+tI!+2!P%?Q65UqnH2Z@ zfvEYUd0o!A54NFmMML;BA!^7QwwFG1pMu{y8*#SFdHdjS>EV3QuX=C!e$b!B{CL0f z*7&x)f3_^@x~}?w7o=ZyZKPdkemU)%a63?7MhL~`lrDQWtwl($JT#ffbC$r&sHFj! zr~H1fZCvHDifO{cxr$rBC>9Z88d*h{W&qp0H`lCh0G-?)msYhm+&PhHza(~+8nqwl z1Lr`?hkYev^)kMan6NvsLy|Cla)o+TJG)rvx|wX1Zbh1bjt|$k^I@1a$S^v?iZQ@& z9wWhK91K`ULRXu@QLi`Ow7!PM#MnyktO)z>BKO)EcVYD55-ph8rj(BX>lPJwYx7^Yw&;m~ z)(=3iUB4e8oQXSN#!o!U6d#G^Pg;AGzh)>yMA>ka zq$Gxib2!g;Q_9+1snfaVE?2cImEUkjZC_jRQ5}N)6AasTL?tI$22wtvDg|GF-3ElkPmbm7 zrpATF2(wG=$HWxPay8x_N@z9+8+8Hri6wz`+HnOKv@LmKvA8JFQWG|AIqxPBYbH#b z%Q(h578rOEOs~+YAXxQCZctHMCYC3cdVLw&>E5v|VAe*%kpVjrs9$W?Jj%_w+jVui zvLk`~fI`P=ylkjhY8PTR`z9f{fK;kzSl>8nuvCSOli z3ye2w_~O)V7n7=+-oZ}1c{@Axey%&ExV{ccADz?LS^#L!tVJ!NF;m42o_CLtqb;r9 zZ)Tr3xR@`Jr#C{B;Hv`xmr?10(XWFUnB5&%fsC-H9CGNPi$4PJz+c7^m809#JxH_A z@`ESq@L`m|RHW$=Nxk`#H4%*jHjR`4$6s#OA)&>)C_V}@^BeSbV@I4O$E$gN3A@=h zucLCPkt8-^LKrar{8$h{3mL=7iM zVwW=vbfmcKMF(c0p?aS=L8pZeR-lx@w@9p5HB92r94P|%Q3kpznH*!Qw*R$NULA+~ z#cWokykxSxkV=(sA!nrpz846Wh?Hqa@}M754O|G;Wh%mdx1NWD@f+q-3Cj)yFLAzA zpMMFIl2Nvi9`G6%r6G+melnQi2sUycEFr%aWdb}Fs!*(2c)Su(dI>x!L>!Hl)(Ory zSP|?2Dm)`0`6fLV%aK(Z32IqKH3NwF*uha0|H4cz@u!gf+?7l^@Ft*)8CaDY+lOhB*Q@W=M*S@}Hdjrw?ZR3-3U1uGZqJ1L@UyGVjQ+j?>pvh?;_RVh znJhW6!=biECw}1KFftjy+o^^?3r7|}a02jZiTWr|DUTl5l=_3kI(|i~wf)cx5Q?km zZu?}yPZ>#XowHu}3lQIRd|3N8-HOufnT^?K@?iiMqJeb;UWBrz|J zFL&M#tnWOCWToQl_owv+XlnqruUw%qtX<6dTa_R zxF4&;?onZ$2Mq6wONCI57T09XV^70!gcT`PVTGd9m-j*-V))=6dhN`8#*A-B^0Wq zchMt`)>AM{0BA>rIn`3P)L{qeajBnU`O$glZ&t7@x27@}7NFI7e^o&+_Zx3ei8<)2 z)9vL1ZB>;36wMe78PIR^Y{r#m2C$?mbl9oXzNk@rkEIx+@*UwrJHZ>{C}X9iUzOcC zCCOL!iRz_rIOZXp7u9i(^1mdr#itmuc2RZHpR?fgwF9796)5koW+&< z^qrcm0P{JjSCJlvVI&~UD8rJ0na6j!=*f{qUb>#S&eJJFiRY_k-V~u?s_LgCFRG>1 zRC@t@%3p9PcpsC#9DL(wap|ZzMt_nz^yp6@w~~I$=eX*Lf%vSvk9b8O$$5Ts)=#r) z=6)JI)n1GUWQGh-^Oo_9K3v6M2UyRFMROJeacSp(HoBug&$Fx5EV#zbe9y;Ub7^bF4 zKhP;y-UX?1JX})fwkf=lJ*aM*3gcu{T#7ONHPusmG6KaP;@b_ZvQ8zs4bl}#D8nB} zXz_@?Y4!MU(e}(g@>CxGPx$rwt@j`Bi;;noosOO9xAn=%K*#!FQ0xBoXste8{6GF$ z0i29QjQ^O|AI#uj{l^{rpqfA6^wG*#4q$CeKtV@uVs7e4Z(wC>XsPFD{?+(@jbO}w zF~|Sk5RCnwQH_rMw;}o8qZRW%($w#E{}(aokM+s&x54?FaajH~I6ut)kFCEQ&JSb% zW9x5b`VDa`e>2l>i2K0HKScM(7G?R1nf}A6t~=Z>;rIsstJ~X- z>1^rc@y)f-zU}4y@$%Tl_GD#k27iB6C3%3}hU`LO9Z8Cp#DkYwr!FJRH`uw*P9J7j zJ=sb=Aq@V!{=!+*6Q#(;=wG`S3AkUKGtu!WmfHZPVVt(fM!;RuF8b6Uiw>HPSAmtyT=3ShC@beAAv!Hct=N zhx1t{R=qEe>Z&YRKAc{R?{qKTZ|9HK{POR&k2j6Q@CG#tG^8o0{hnYhUF+ZLQkAS2 zx_^W$?Ngr^>foX!)adh95#J@I#45Q^#<@p!I;VixP0)dYPztT#ap|9z=_yI1whwpn z5AaFd@lRkRn?&ddbWF2Tydk^M z3?xHVgJC9rVGT%PwaF1+G__s=9(3YQdt-ZVs%e>C5Pre9P?ap`=Sm~QF6)nl6dK{= zv--gkyCJ582(QuiicWB`8Xs>Rl&>-iBB^PGpEjUQ5x(-0$#(!fPtFPjN zUcR?bYLUcCoAP?sd}@UZLxmt$&aTCb2W%180e9a(1C5t(eyRJgUX$}nSnvcowcpzh zBod(n>#ym4SP@1;-!O&iT#EtT--H5rQ)ogrEH+Z8g1OWg1^8s!AK=h7l+vKYoc42(_UEYOmTn@=Q|)gJcNntVsaBUH%1> zNE3p{$E!RqMjan4ddCQq^C-rar`+Z41>V0MBtOnCm{4GBOc8pf;z z?i{F81HOZdeP|x<))kE@x`-zqL7!@zPAZ&DSphqE0sd8K7RxO{4>{QYwge;FTfr8W zbY;->I|<(}#|5{nHhqd+C)E{lT_!=9IdzlL%q5FQwcv>FQfZ?DAPu2O5RQHxRMW=E zIaS!D<-ho?Ut&Va!5PuTqLT*VGGL71hNXf(5_q>syer)3NC#=`wKw6wS!H-oXDDDA z>Le7lm%xf|I>0_}idB(bfpc<@eKDgRWbE+E+;svUTgAR>vdR{jhniXnWA_F=3E-9m z{_My}?IxpRha%KMc8I~2ngm8eruiAdYN>YB!8aOq;4uU)7~~5q@8#E^Ianbwy&b~` zFcD5plCWOKIfbev8UVOCA~DOw5(Ju5gmqn^&*Pnj=vsQxWfrz!Q{yFpNDXyZVT6I*lr<)APAQi(p4mG?PgE$1yNJRWlL$qX1}sHC%xerI{qu zOGA;0AtCkJLvmGB4f^(Fw|p&3NBa+q>l}%QST$~%u)2hWE`)qi9BK370rmp4*dv0~ zo9+-y_<|&URP%0j;)fcK*`o`;G7)7;s^yy>`8B$cgjQ(-i4Hg0hK~vj)N;qQ0g*1x zNnN(1Qmh|kccN@1VZQO*mBliS?8clxzw%XnqW7f3SShDxG&+r-eKDZqGx|;{1s{iB zwr+{KDn1}1T#8X!dlvI@C*jKfo#4BlM{B;Vs`XYD?aOC(d`ipC@nJ9_*1*hE~$})GEqyy_s)poZi-1@wv)oWl)}ap3K5|K+bWCOlC{`Mh7&#fo3Xp8P|`R z+E&aaoU2>k5ETbOPje>wY2k3R<%sL)f_u$i{8=;Ac5AA2>jGyVh2bce?fKIlDC+tcDAwm^xLfGdWyU zw7enuAEfab{I1e^$eQi6Cze{mrOl**59nAOP}Wb7+6gH-q{qJuyEUB)Nh0#=1=(sw zrL3Utx$Wfw*P&N)xLPK=r4eh_%#b0(1vIUF_oCl>XkVZ|zI4)Nw)1MPEk@mA+VVNy&)P5mXZ|o#XI@LcVKEMD#!UE zpKN9eeUN1FXdBOD3SlKFO{g&^gAD3EES>eIDebZ|kZ#iDsM;WSgjyg%s4TKe3Gy1i zer%?fR29B9-ZBYLcDx-GVMS?YW0zO%y?*J~@9s4~a~b1&Y@{$u`-*7$q=NU2YW z>Q@Vp9~xKk^@$+-)r;NcwWQCnOKV)tfdjvX_~fW@MEi45YwwIh$ml$~SPl7{A%G=R zxNiZ21$uGHOg^En8u_tK?7U@XGv2GFb`v7fY>yqBeJbJmSeLfWWM(SNb}j}80#A{; zH}XlqPQ`}=?6~ld+M-X<=cKwoZ^;}WUfR_ttPn~Ox%Ya~WpYRVWTU=WOyK9p6(!j= zE0!?;X_YUSd#OxyG1P|$e+b>y1#i(6idV|99xzqgq=!3OOR58{1P)OwQoRVIMQ~G}YMD+% z#17{5yp;|Pgd-o<0d;^3M9TS<2i>MU13^Rq+955reo&U2*kv2>F|unuMmE%_Ll7d& zs*zjRN2AWNu6MPW@F|@>pKCcS5dPR?E1H4=?2SZp`Os?>|F&qV0`E)w@H#0mA;jE` zav?u_32CWB9K#k4ss|cV*6T=9A_MB zZZR$ecoC(MfqEjd$Q*ttfJc5SzU%~wK`Ql4xf*qPw5ZuHt1D6qQxL5fP6Kr@)iv= z_MrH_frC>D_OGmR)(Z~_o--vB4CBj`T_xl`+eW=kYgJ8i&K{9i_tlJKn7VYl`E#uN z@3d|H5Yuww=hL90*ydyoKKP#1_pM?^3K51=KS(-?sKxmVkFu21-+tZF$DsQxO@}PSa0y4KuB7Zv0Zin$?{s@oyteuvq!nqm){>cFx3>}~Q$a)S7E+norQuC^SA-C!uWQbt5rkjf z&KFUg_69yx5W5d@EjG#=HBK~-R~A`5P(aFQ&Hlkl1~pTfVb>KoN+a4Po18VxuW>2U z_4eyr%Z>!)W(L>oh(eB++ky6`A(v^?{jhYju)<=&YmeHjd)*AvHjqg10I~!^)BvxJ z=0w);N=+AhNgCFn^pE|8Wa%zGeP?S;eTWwr zaf-9%lo-s3qU*CeB%sf5zIuhJrs^pG1qM&0J!3z-FP%N~!!yhhgwe_-Q>N5-Iv z;zS8^e08D@qs7oec@ZPpxgtW=mK&`b`ZyMma(U2@(~fYQeR*F}XwiJ3VH~A85go;_ z1Tx&-QxDXZOyu1~czkEu+urzn;d|QqW9zy1+x~f$`VhKJ1i#DPB;Z){OAxV56=5Kc zbmGPiC`K-L*oN+pngv~HyS$Vd-kJ6oGSkgR=-01;VRm+&Mnfq*t}P<7wC7WO zS8{}7i~_L{ET*v6H;1g1ph!{3J*lHcM?~9qBH<(M?n)7Dy%T9K!=j}c$14vn@`(ci z!ou6rDtu9lxWe{Hbu15A+XVYlP$E4B2fGYdmkkLz!cR<(Ku1;jodY_H?ctOZ*Ut}VQ8!H9)}YkVKDO{{OdR>+=FiWlZ*SzI5Wa*rf|GZ;KjDAL zcTnA$Dy6G7hIdPzI4XNT=+;v z3$3k*6wTQlN946+<;EYS)%uG_^X=*y`}fZDx02> z<$}AwX*zgy+IU&7$G7tDqXo1t=OI4FR|~hefQ}Sm#6`bHhD8mcK0Pg|1D>2h-kY)| zd>)3X6@VW2ZbI+`u-54=4YrHK(EX}5lh{>_+@?L{`_1cpqbN@}?KMg`Z5;najba#g za$WBP3eyS9N&B4QuD+S7%x*$2-0}A01@28s=g ze^S$bOQ8RtrgLyG(R~D>uycN>>6~!_BV!vwV?8G`2V+OGk32O! zM^|HGyZ=E{)1RC-rN6}a{|`hpvHY`C{#SO=|2muKcbd^Zit2yT;*|cV?f+ef66?PS z+W&7T>>t|nzscjjRdm+By1^t1f=7$1*s+*X_ki0NtJT1a*b=xZf@gxX12)gVj~)@I!$VnyotxqHD;3x#T>SUBes(% zzj8bZ6ax0$!7Lwnem?$e;klC3B4;Z+_94?3+3z?VINl*wUth%XqVyAo`#nsfX!Br3 z$cS6c?s{G6LrmW-#dZc$_|-?-Jb7POybF@vzP=1TT+#Qwx8ZtY)+oziSUVoq3H~ge z79_Q1{Z=4|Gn14p1XR{I(+NqbC1~pZN z$B;VUhDuQ7#4#N54Rl6EH($@|R&xk$*c}xF%M$8&|0VQDG-@iV#7Krh3%4I^--zBY zrI;s8wNeEKRX#hyiBayH5|PS{l&AVxPk{WW5un?-Gkj7t7z zotBEvhMorP#kUSDZ0dK}1{SVLx19bJXY3#xtw^jEM184BM9z+rEik{I(iW$eFM|;l zx5b67c_uC30xiI){=4Lp6pomnpu|C2)U^4#uj&hm7Aa9QI{xhp6`Woz^deEIQ&K}2 za(noKFcQb)jj!tVtY-gFnU3$qM7t`aUbU2Zsb)cC9RQ65Xv;s0Fzm6Re8zo!1Hya{ z$9xhMt;C$jP4NN-BJ)YC?TDWLuG8AOmwLv|in--6)pa9giuv$HhZSYZ&T0~SJri4zz?v#_icgJT-@wcO0Z z=_+@C=!^q6GNVfgbYdRGX*k-N43>@;Hd1ZyT+s$VLa zp@7T@+7l>+mgZk>m@|KVv0CHLT4c*uJxr|u!&Kx*8B{h^HG;N9=7iezu@|S--*H4A z)8h(efxLjVgYOZnhk+>DT4n-f{mAtpN|O+DN|m6^P$7|8&_iC#EcFr`PVV62RtArJ z6a=+OwS;US-4$e$A$>eJhcpAo!?q%n2bZ}XO2W&ke!PNr6V;7a*2v+V8{Cvgl`&_%k|}-%loEHu59%L1 zDu?K)C#gQwZ2aJcYvl5U5vZJ-&4%x+U@E0C>Kxa?raS&BlZuRP^`p14bd2iTl5IS~ zfN?Y`zRs6ek|7(_@ zeItB75}mY}?-av6b2ATnltn>?5#Z82Dx9Un{>==6=AK*M$LsbI?%Wg)IW#?SE+l_< z(`g{Pn&kqDebC=MaHh0Kb$U`IxUun<{RsC&=|jnSp+ zTmnKhpiLx}r*ow>s6SIcWTyF`7_TUf&i;1fKrtDa}O6%9;|*usDLO=1F1pRN#d)^rFlyD(rM}Q32KMn zE*KZ!r>(%tEv8yqIH{F%AF_%=wklU4B7VXbShWXcU>eOhhBNi_Rh})KY?%WePF1^N zX^vgUg>Y`bC%ii&DusoZM2>Nk3Uxx!Cz&nfqM5y8iEs5Z8)~ddX5aQbkdru;2dK*P z4;o6Ur6Neyl#Aw!M>wo1b{88;;-p4lSA~a*Z;P9d^QK=5$FIaSE@$8pA{z99$H&tX zPBBcwW#&>j+X7fue49kLZvuybX$dQ|bEZ`~dpCHz1$e(vb_pvdCeV)~Z)PB|5G9zB zl{NP1y#9jobB-6Ja7}jCQCrcT49H+m#G<}J>`PMk7zd)tMi-`45nY~&d01unGDD6vg|F~tNG%72)BoE2SGvF0P0NK)2lSObh4iz1c{l2|x3@x<__ z<0*WnpfD)+t0S2_Qo4FEho^3hNbhbw0Z#yJ&6dP9l|(sUy1BoUKn zR>SbX3?H7J%9yfW$@DC$c6HlWxBa<3&wm_gk}77|_WE;=w9>k1r@k81>m2T!PsmRCIN zo|Vrys@@_FG^io~;_C2FwcZAspT+@ph+Yc|DdM`#y&?)?;%m{&=U6-5-9e~db7oIG z2G~R**-Ai#%mZ9fe#o@^)DQ_ABPgsZp3j!+usW&-S-&B)lI|ecpbF9G+u|i+Os-RA zG)5Wn543O0j*=qhP+9RMuoCVu%5_-ZDHX}UcRr`fF~;XzHWuT$j8E>>O#eVfJma?x_Gv-NvKk}{00?q)G~;(+7+!e7l?x6CG1EXf>;cMMLOR`V z^uJPZXJ-NfHN2z>)TN&Dp~=#HW1v9-h27pL;_78LgP?}R;%D}`f*)1Ur8zZ5CWB5v z?y4&B`t^k89gy6mRPC#D@eHS7oQ)a^v!+8hrcqkOLh0NY_VJB|(*pMAI;Y{6M{-nK z7(hO}g9g|^Q3Z_3yOnNiv73eY&xZw!lGDc_g5^?b)UZh-ng|R=NKU9q*L@t@BU$1H zg<*st{8{=;_FeE#P{8e(;GLE{IG5V_CQe-dgk)80WSa=IpgT<{9w{hPuNGGR=poIz z@1GMB^{l0BK0qad4kIr-`?Y}r2+)>EGmM^7N|-$ZT!u7!EF;ad>P^d&DAi24*aR@q zr65uI7l5Wqm_O?LUNn`rs9H2`82K~PNG9$AsAp07;5%EmNMOR6)IE;z+Dcck5|hoCogG4@iKd;hagZ^Z z?$q3FFh_k4w`f|%bkDLsdYdL4NKY<;itq+1E2gMGmTghcd+>fpHRNzHSp;t#m)4FD z->tR<*d?e4#*ljkULI4|4M5zbo#b?juacU zL$^gwkWwC)wV&<%h|4`@ZlHhsd>Mt-7zVDu2STE30 zS9U{V6(`UKmPWb(l(EDZnV*k6?#efKn0<>iHx~-v9(&ehLu?p`Bb=Mz_pyR|>2C?7 z5I|(a5i_?@>ryMxe!_v0-$ZWOTe$`Y@2|+|FouxzWNEbm~9!x-{X#cW{Aa4o+X%I&}-b6$F>?-66 zvI9OkwtJgn-P0b9GQs=(sD!+3cyUU%P`-DM5+uO4=3_2Y;mIu*7c+g#KMiL0I`e2V zm*j3Z%o5R9|-n8v9kpx}bXrW^m``5Uu%UJai3 zF`eUQPHYU5=z&EN(`d0Eeh_i)_Twf!pWSTxx=C;2D+FfNuYBDcsjz3v4{z=uJ?D?K zAl<}0W(CSO3%+qEx*8^P2`#6(KmoVlkR_#+=Tq%IPSFZ6svj$*w(Qyh^LR3}sI=+! zV7Q($n|Zb6jr*qll)uAFsL%XFxv!narkwA-q-q!ujO@-Caa zyTH+S901vMAM9w{odf8L^s|~wj>3cjqaGoEIG9Y9MY8{#wVbZnL9w52CHJe$LI@Q( zGN##E4SYEzu3u3wv0mkYM>K%pqE69-c;Vr;%cukc2C)YZJ4IX*n~t5xbB*cHJ|xa2 z+H4@3M^vzf%!!?W4ndnl9o`zn#<*R@g1THh`!JO45xeF1#{A)iCxWv(4}}Gb&s&;d%1$CY+zqdDmLbXWwUW>)zj=RT`r=@o3g8O z9zn~EOnyIUu^|MW*XmQOkeDpHBSoTezT8^Rz%vc}p8isCGB=Fr^1lr1fzer$ z1YRA^8o3|F<#1l}zHmP^u9pjoCB?AA=P57g-TgU*J0y8N%v4wDJZG=*8hYsQyDIrJ0FTJF z-%FUn1gsYMTB)5Vy2^M%HefkBZ=`K??Y%!#BX+6RW*OC>pC+!@s$I{ zAsC`g!f_vme1>s2QJ3ZzMqv%B{gbG^hOXAsk0}LCX+i|+JF`UDC6HfxKlXuP zsMM_35w&tho*0TDlRRqtmG~V*_I$^Y$Csi5vjm%e1ewnSq^e^cK}gW&bv))l|WSGY3^?r6b`R?gOK6bd5cJGS6-qz}~=IXjU+;zPV8`95xg^M);_a&aIi}%p1GbgT2${;q|a4MLGmC zeYLUWN`1?w^SK0*og8usU8SJ^@JNTKD~Zlle2m>X~V`{T7Uq4q`0 zqry`qax}*ZP#ith)vqA(pQX+O!?=H;zMRxmCWtmYlIYJl=OBD>0Tm96xUTM~q*p&{ zPbk+HM3QaKQs9WuaHThB*C;aHCmc4Q8htLBuJOkfmdqSHRXnsP;SXSlsCNL0)*fdi z%H@B|G$Xf1$}!&2k~WpC#BR*N@PTnX@zFx^MOSLpiHY0m=q8{q>np=(hj4#c z(dL|3CP~`xb}2#4t9|2jV?nE5DU(9Jz#yeY2)ir{%v2A62O?|Qa&iKv5 zIq13f0>4O9-s7KC;oq^a|DXzUGBD9Ga58*`=`YUcDI%d-6)1^qh+n(g0%pnq@tYYa5ozs5kb{VnqTE)B)@x5)duG!)yv z#z3?EYYa5ozXy5$zQO-eq%$WIJM({%f5&xoYe^cdy*fJ$z9^CM7xmi$dndf)P^lLf zXiHsfjs&!64J(zGyA^2o@b7207D}#Xa=YH0$x)*D1D z19m5To>qH%*%97((D;L>_;!^CvV_LzED?F#hZCgbL_MMHUyfH73*Y%(cW#CYAMRoh zY@`?=WY-rByq;g?5^HS;Hlwh#fTVEni*@{i8+ta zoQGzvjfgGXbWh2G|3=+cM%B@6TjK8SPH=aJ5ZnR;2@v4m?(XhRaF^ijZU+nQ65JgQ z4#Apy_q*@i9#6W*=7B&L{7%2mKfWAcwctMByiCl z3?|c&<1kKr7hpA~?>$t#?!-uNwMH#=FlVScOct(4Pc29AX?YkRyKYR11F!J``SH9C zie~`r+hIyOD!ar$`r)FdT#V)~<}f^W! z6qadFx}w>SpA3pZ_ICflaE^ ziz??sxyZg3=53U01enISV21sz-ofCmdC4jsVWj6NHu1$t$kO0oqk-}Ll+@Gge1}W7 z<-#pcCj}2v)xSd+0SIi2>Z612r_{!|V;+c;>pl)dttl-le<`yahA2?hDe*zK2h1c~ zFe7@YQqLi|>OOr4QL^ks#lmCqUUv(#?yZ1&dL$62gj28095xt8gBV z>4q2^?8 ziAL!nwPe%^8ya$2uV2;6rYZ5Q#p%ZI(?=T1qzXtus?!8M50M_AoZFWb8Lt$T5`$ct0HuHveS>unvPzlrYvxGd&ZaChT#!0jPKuaLq1XuhzlMnC< zjgGQ5>ucg?`SJcm5tbx#Ql=2*_JbTF42~p&RPCrby0)95z4xnx zA@iX()({m1ejz&fvjZv$a@r8zF|okIpTSskx-Tu<$AzJ=oDf!k@}sRha?ch8nf@PN z=l(<+C4Fcl!7~GfT4PgInE5AB*~(0z5{du3Kp0H}n}cEI0ZIGVqbBdkSe3vfV17~# zEsTL78z$#Ud<2FSvI9`Xa`@vx-HC;{-n?c7cM#TsF}Spl*Vh?$Zw-Qw9#iEl0Uf+4}T7cKP4 z{3^*;-8Fro*zL=wCZ&o|B=18b)CZ3g;HMKukPuJiHNLFpHN_#k!~;xq22>;QVYS*- z&#~`*<}?zBmLHfYAo#%ai03bxro(I_T{w&_V7yD?>Cf%(W9x$t&h=P99tmX(@$mw= z3B+>%%jR{Q+;kknGpXBV0xtPKH&a^k081;1%Ne@PDTIl5>840)<+JfQ7G0wE+o#Zq zh>D(KjM+G~={S7ofOvB{SmeZHuQ*Zutta+Eb$xvK#v;x_bxxxSoQtf^RtVq;`t#C;^hnUSDi$2bzN#|o)QDbdj=!2EXsR4GG}!2{cp>9v#YGaKfFd-eh?fy_#V!V)?=UNcSR%XM8g`?CcN6rCbTqB{1B2pa#A&OV zYs-^mYIBh2aGBC-ty+Yg(H)Fu;$qRIPp}Mwc4pwA`m;8}+sKob5{}xpCS8WA0W)!^ z=waW2rUwweWQo=o^p_K^<|fPEGauBAw6dEI55etX63c1+?i3~8WGNGGlaNycZDu-J zIM~P&{8j&_ObKA%xaGkY$yeTYTB1@_7t#7x_<Cj6?w^Kh z9RFuoEIOR0M-NdrB-M9L0NHVz(R4yuXKEnvhlanlatWgY5{GzqiSvh64MmDfMk*h_ z%sp2aHJ~J+MDk|Mu0Zx9%T9K{PhikuujZ4f+s#Ur5%&?{;p;m>rCCt3Qx$rNdS!TK zfX9N^zS+Z#m|=Da$%&O4&uO zRV%(Z{=Py*?S7x20 zpb6OFZ@d0gJ9L*$vkQc!w0pt=Q%l{iU*d`vTo} z-@^yiJxj9lvdWJkn)Q5k&-BP{SKTfAavbzhwOvqD>Gb<>R-;416~CQv2-UP2GaOtdcw3c*yMOLM-gaK6uMypp*UE z@r=$yAg|Ty)7c7gwf)1?`qs+Hr^|&vTYbtoMs9(%pE<;JX zpN`r|a|clC4El_`_m09MA_@A!ahpFX9(N>T@=Vd!Bp5~)i^TGKm*7*DB(F6_qsxnX zv1~1l5R!t9kM`&_g!(6F{8y^TwTuFo9P|SeD#!f8TomFpboE0%z_QTzOBi&aGgyc{ zPA^7v$``{Sl1)Nq;ZYBNA}iA7yD(v|2YhVQ-VpA~VX4UNaS-{=ZzsV1<8K)FL)c_+dA~A`vTV3n9FR;kGahtWwS<2Cm~z3rb2sXegm=lbW4^~a3+_bdw?TbcoVdUKmq#&R?mos`+F)@;< zI?NeI^ouDvt{3m$h3K6dNK{@}q9TEUJiB$zdTaikA5#xPwOdRf*z3DF{K4y)wOY}@ z8c@wjq3(udu>s(BP~xFHST!DqEn=T`L5F*Hkl0oTjkn0H9>}B3YTP8tsjP&dfRDH*ah`557Iu>@bvY z(Vc_cE|95Gm$VUcVMLLC*!(^)ntzlqjADfR*^AXaa74#*Xo643vFqd3=VDfhCzP^k z!_gQ1v%EH2Q|2xWMDA?^X9G~uJBeA|Yt4L2lo}U`%AYyR67LIRWZMc(IYLjA9v;6; zkA?n9=io?R4|WE8Vnj`dKO#!O-l(vlQ`&(&o>vJmG3;G(+Xldja_NmSBs>flWQvPF z(LDvQ$^rYLdlCAx*AaoHZUb-Wi%!iR*b$g7!l=K7#2sNf($DfW2otUrj0mRCH5;%Y zV$mFp!$S!4EkMT^$3%@OTz9|ufRzh)fww=C4uURpJxakw(&5RyfP5-r>P_Z#RP|sF zRFjvz;8bWreWUY_kHA##c599`*@b%j*j8NqTj=EjL^5C#v;P9+ApTCYH(ii-7oLXX zx_&-9Lvyc|YO??vADB#PyPO}UE+M2tzmE5;TZjeB=ZSr%9rT9X$x!vm>d(Y-Pwwl4 zV6IEpnrVJq=RYEgH{n(^-9hMM=5S5~!oT0|OL{(UU_p>dL%!TJ+XWkpMo~yi*&Ht) zuk{G|Am-TS4`nBIXFtZUl@&&3M^Ahl`P_P@#NpjG)un6pW-A$*3_xZuA`zCSrZq8d zTQ9IPj9pC>3g{j{Hc*!i&62Zak->of|vB10yd?fS=t`#fcYp zjzBwJvc@gcD{;1A#d>?r)Y*y(7^MDjzw;CO+soo^^J*50w{!vGK3xda*B)BI@ktzD z*XZ@AK0vt%%!_iM!Acq_vk})Wp_5zEA~tbdu2aR$NB>L~{x2QnC||9q*fWY#+Q~)q zLKoe?O_Rsc)r|KsqU=e)M&7FdP&(Q)cgXG}yTpPu+=VLpplo3@ha>VBW#1BJ2cFJp z4`Lf%jkmLC!+_=dYxst<9YVO4zfftc&QK=~K3ZSjIh9(kW=1rOKV2v7!=CwzUI%uD z-A3M8+Loqj`>Hy1uGtfQ&f>Cp>Z zpkJ4dpVq#Le=l$@R1dy7bs3 zBWDi{aUG^t-!KB7_>YY&R+a+8aXNM0a`}spa>GVt%O1Ztl!Zjqhj_=NiO*iT2&#F8 zWo+zFIh!Vv2e7PZA3k8za^j-WW9#k(OEx0gr%Gb zQ4V1Ah@`79*J}rfZ!S@<514#msVVW-i9^}EU9{KH<0|?ow;_seG!kEGnvYa^XG1$4t<|1HV~fL%lE)*74<0^A2y;LH{MM3 zVN?2#S>m9Fu?|PDEAW;H`q5q;i1gIQRJUZEmVG6c*Ytsvy;279hwCuNR3P^;NfP#guKil`gFH#J`e&Dc*vjweF0QSB+w6pIs(DIarN_SE*(4&~@ zM*DQsMItA2LC&habMO^9!;)@*7>KsstJ5d1%L^jy3bYUA`eBKVxLi@tV;1a93Re^; zlNWc7`&txu&?hV~=SA%x+)Y_NqQb7 zx;98rdI*Kx-Jd&st3zQ)*}sUy+*H>7X5~*|srGk}bCM0d-P8^o^g4_75_dX&e><6` zo26=sir{*#vlMZiCogBf;XMUnzlJ<}5$hf}@9ksFN&#AWKN%AA%LUxt&*wqm*p`ZK z`Gt4|qEuhCxjc={z_KgV0~orfG<`LQ>#}hx`UJo1*bMWmZnTe_m=oiq;yX29%Hz8) zbLBoA*I*^Xh1tY;Qf}k|%R8zL^8KGsAS0c{8Rk-HX z+GrrnzkRRG)4!YC2`g<0dH#GLt+e%-l03;EJ%#GgtJ-jlT4?sPDL64{TgR!i9jaOZ z^87%$e(PiTwbezZl|w0~-)CtssO=VN_5uqPhbwtfkBFK05-=YdlnZLFq~Ir|V;n=; zcXt@ZK@5e5U!a@s??FyXXUy(J))g$6q3NwcKB$=pZcPJ^<$@0_xw>+y(dgccV7thA z4DcEbiv5mfy8uS!++{f^T9FjWlx=Ion?`B|M?U@21X`yk6X=XnevT{>!?F;kh}sTy zO~0k5ZTi(XdElsA5^({Ti0E8pJwl^OAud&jsF&D85gMcRmBKdCVXv;LzLqYV<;dWp zR3sSN4V%V(zhpUE`(qZ}Gx|i^^j1^(H}`$W##F`u=IInT+^okzi^nU*rmYGrz=-p!x-!oqZgHGB0=GHJk?avKo7|VA3|4{kV(TFxv!8T z=3(`p7D8Q-fJ>OA9l4u}pBgzmM*5KKHrWDyX&rIo%+?GwDRYDfX(#ZZHB#{cOV69e zyFhz;{cf1wV=t-lPw&a|U0Xpv81KNX(iU1_wuXfLZ2NxG-7Iv<`fD(rfAodKyrv@o zD>tyl$(iw)9fJ%!W(VHt;&*SUMZT=VAjTpVA=5bOTR1SbZ9qC{$kF3`xz3(`Xpl^h zYKjP-Oep9FXRPcu^5~xAzFwlb=1kV!D;EqG?tzq=)~K|bi8`~Ib5)ft5VbUmC0?TSIefN4iuxj9q?X7-CM+7 z*On#y-UA1N@R4j=!{W7Dj9s_?ZK=OxL;SidKplvYYT^7hMfjP&LX-SlHC9Q!?* zS$OMJ8qE?%8Rpp|92dvg$0jr5ibcrV6+c5mbC|Wo+Y0*oUROqqhRP0|!5z70vF!+a zcRz{>>Twr1^GjuYWyO=cwSuNT>sBpw&cqA<$F{+@p{97$T#n@xh>e)I>Q@N8F>)M& zC@5QO1MvG^L@#mzA{^7B--1K|+<7T$gQt+L!NZUiRxa^Bnn$4}SQ&&Ea=d#uq0bqf z^SOkcm1*2c8GmWtViH_EYIRQGWS-6iW#H9>Os*VzC|ekfI^CyR)?$9q{177pa|EGR zrAJSH-9TnAW72o_qe)X=bka-%h6wUP=a;8-9`mMpuW!fl00RMId#%WDDnsfJTOr@u z`n9)hw$KI7#8_01PmdvqRWz$BW+I)7061g-4G2TAVLBC|hpuy_>57kI=maiPsDlY6 zD8{p|G-)gF)qSie*@wWbW#R7r1?o7Fp63xSdEj*vWo{!~)t9d3?G@rS#bI}&Qi_!Q z^LurU_WzwA*?qNVJZJ+~R>R6e6u?h4z_oh-TU2%3hLM{8p@7)0F63Me;=TF;CW5*I z1%>9~Y^S!=bd^qOuHYf9bQ|hMu2t>t|4blPMMPu}L*}xd{O8m2)on499@G>a%V=5m z`_nTs2h@iuA$p3oxSVR>Nk{eq`6>DSkeU8?srJIvj_8ZGGi}FR=HSRR@*@nf?w+$m z=nuu}JHZKqn^nTEj@O*ty@Mfl;!jekUgtS`ui zL-%V^=)EH!&T!!LJ<6B;x^+t#3JT; zGYDMQSP*l)y7B#AgWAmG7mq~_NRGxHV?u4S(+KNfhBouz!IMIPHuA%Tu-p5rNOqC) z`lqkLdc9*cX$NK7$wy;D(5Y&Z4{<79@MDNL>my1xm;I9x!<4ZRfE|v~Y^CXW-o>^( zLBNPnL*eg2YVBOP)c8ran@ly-`b^x(ugAEQp~+L(2Z(z%?wH7MqbFozZiedK=+~=X z3XVg;^B4kv&T&R;EI^_H{f+8;zo#|WayX9)1P`g7Una8(isNGVS#k0VDP6qdniEzx(g;Q?~R%XFraCAQQ|U-(pfS- zDu)t-g`^$$_(-bM%$}e46F5lVhcy0%z(Wp=86Hp+eY#7X4RRj@N;boM4QT>HTIVxR zVKHY~}u%;CFUpE&SiOZ7n91+vF@ZE@NqOGFY7?p|xD z3BgZ7?C4}frd~K!b`TD5kV$7vq>^f)e@q*vw@9)>y3P|XZ|bjc567vXIW)Xn38xN0 zfMDT< zXjgmrr~#uvB)rd|XNuW4YXBkAFrf{ITD07}yoPvYq#4xyTJcGJTSq{|Bw2$}s|G|# zyc&nI8T!TLDFXk#LPkw}a|nyZ(}V-cGsg+s#{^!xM;L5>HR4J124g>E)S{MOljB^f ze9!>h6X`m0%9q-}m&BE>W6GJtGubQM)bdO$XW-US|ImTCQw5w(7YPH#@j!G6@h!3;N`ax5;c7Q0H?Rw9u)uYtuICE?DQ402ln8?3 z$GIzbIsQJ}sL!Pl#0m*$thyN8khG=;JN5gS4F*=p()t3_nUr-N+t}kLo&J586sX~@ zVqL_(>qerkAh;bcj+!Ov67OXq@wW~fr|lQa$+Az>^WGBrl656WN?q251zGCMB|HY# zqnvV@Gv(|Qq>;vp2cEDPe4Cv?j+FG1 zb*Il(C@wYtr8VGQ5c-~*d4>5?HvQ(5&w#0>5$n93M~m<}9#Qw)Ym1#Xe-+dO>r8X- z33yhe>(RoZKuE^c2)}_ba}sjocxdtQ&gZ9=G!u7q7qX z7(C7^qoR~M##R(#;5FuDNpd6Wuf*mAq!r0pm>l z`v<1m8+gwGLujN7+`X@XoqYA(MN5;qzSodeOQwgnYx2J;#ww-POgZfZd zZ^^%V#(pch8@jK@!nyyk4|PB5HiY^jqE68A-aD#j=cVrxingiPpa=(MF}4N%Z^^sh ztC4Iw+|gQ#bifSOuTo-xQ9)K&-H0_YA!||XYP#)r*7va z;u3|DYrmMCR|{=8fk-fz*5LN+dKlrC{o+Mi7AAiYp=S|=)S`(0goXAO_3GbPXaGm! zzncBCNGQw6sw*5Z1$8au zzawB{{XYufZ_3xdguwnU%&kBB{Nvs5FV3C+eF*;`js0r~eC zt^cYlAJ_R7tJ56+t}M6LgP_tpj9{#b*fk`wjp9PZ!)@rOoH}0PhxE+t)-~S$t#7g|`j$&RmKd zxw)o)vND>EWr}BKB&DmLn)Da$@*&i`PGeskZ78{ZP$(TI_+$`9Jt$+4DOY<0%sA>U z_Elo>Wm#1!%yi81S$>(M(zK4KpF6T$IuP;GMy&lx%M}>Hq|7x|Y9U`aI+f>a+pf`W zTYRs{$kf+;hM0grabP;buR&vY4?VG$vgY}0o2m;nrfSp_XJJhan^8A`p#rh>bm6Mk zvEKP?m}!u$`!*YUek($@VaxT5lHd92ai3G|y77C^$9N{cknf9{%(n&b_J}6feSUGB zDE{KOA7M$U%m!75svM!WvhyK{=QA#+{QDD=&M|xg>};V}fK6O<*b7kqn2cXy*@r-P z=R^ASi3z8eMia+qzM<8G@1th#9mjd6d50|2ugI*2swmH&bxcbzNANfxqP0d+l<#+S z6}8eWnZ!oPY=)`|-bW8g%Gu@SG^v}s@@j=!Gvt1nR6;k+3=r3pmw4(|am-PRZKJCx zl<&HrJ74YkFein?Lkp1fpDj7hW#ya@IjfsQ{&E=IykI}hUS0f_8mrs48eYn1P=wAu zkYNzWl}O7}|9!Xg`Xy7SvF>&+4rMH%VL-bA`Jr*ly6E)O+Oz|x9|9&V&&%3?{}>f) zmY3xC2y&eamE=gJsVHC1uiep@CCc1b8g;h@C zsiR$T3!*$U3fdH%3fha0Q8KO*4@Q_;AJUe20w9z@NM{ z!w1k5WfGLF*VnoMcbtzzc(+td;w^f!EuYa$FlKS{Ooxk@dNPqpklH>Dm zKHjb8uh`XY*S!ef;ZH%S*AhWud5ogCW}J{C<2hKR-zHFCO&L^V5z zNCl`O|0Vws#RB^R+I15#dnBH)D+6W(MA?`WKJMKxWZ4q=SXuLaHri{JO$j#R+7Pd` zOYTK|wrdQCUBmcIUZ}}ssCk}(Od0e`cTojV_pe}~e8ENP#LSN9a#5(gtP1m+EB$oo zebnHCLDc=m^9d%_@B*0x?9KUhhbWY|II6k)+Ho$Tgu8MO)zfYSy=5%n7WwmNn`ah~ zQe0emwFDS~M3!kF%&TLJ{;fgeg@hu6%HfCt@~A$EIxcI~XpY$8heP%Ko^r;5EHf@E zJXRG8EAIK!qJYALyb`|5R?JFOWoCmqq`|5(Tbvng$oo%XCbS^@!Iir2TYlaMcCpZW zMRv&MX@c-Guip(n8+9&8F)x(^$A%pSk74Tjr&*qOkwwZB+fx+T{87!;@;5+h4-)ns zkqJ3dHoWr3PJt?L2-#=_>9{fYBN!3p4qS=@qCyhNydOrtRE3d87(Y8fx0=Cg;_PrZ z(ay_%Lu*S&K$u`K;mr;5=2b@1a**DZC5?z6t>#T@6)V{45DSQSu;Y#TU=bFsoXbL< zblSf#ay*I<4V8`_*`i!~iVd$4ZpiaoGC7Nc0Kf+LsjIlDiY95E=SBwX;prFaECS$? zEM?KS3)4cYv2ZwnzP*FB*A-luGk(8H|2u|xL#>@H%zxP?$g!7&VzFlR1!FapWubSW@ zRB7mi_tVRG79dDTF4s9M#tmjdqCrC z{G)^knZbI0xapj+LdpztwR{x}?WEpkGO;6rd`|Pp9IckD)#lz=>SbMJUJ#cjsYR87 zd7JI{m9G0rM<%8-6e@#mnIYw;t`wcCDvA|g=(pi*zj3IP>sIrj|;;L*coD6(h~dNy8+uJ)I$AjZii{7aE3u_0qDC2%7!PjC?}_M z^=oI!{t)Po-_F#*e!>eB3SjRL%%As*Br_wDQ*AMwfS;cfnG@XPrTrfIl!r|IWQ8}#tmdM+$|GG zSNqUL7!$?Xr{Q0Yn)%$7;YO6*DZS(yZB$_0WnkH*O;*c>>L=t26=inN<1>e01G~|7*YbuuR z7pGq&vt#N-1KW~9$}BkKHe*Aly$8_0-8XMUyQLhMM7kb@V#5|GUcI!W&n<$`@RaFH zg1V_n_D8-d)JfTWGJwbTp@gm=SrCNHCXDA8LN{>_^y(nvNdf<$xlk)jM&NY|_lr&W z*98n7eY_^}KCnXyPmN~8bIuWH!1ycqJrn#bqyvW_6|XC1Y>OE{we(_b=zyf%+L8}X z&$p-y?!?klGGC<@8P4PpG?r!dAgtUd`NE8bc0HN{I59`NPcKWr7MVOraxjOmblmflfYA$lAd@?}ji;6TX z1^Yp7tm`TL-TTpXM9MZ6c>w3S!mR=znFj`E#S+axBzD(R{?|XxB_dzc)#bq4rPZ0O zFuib|zokp*;CNh2Bt1Uk*mM%FBX`1Q;LNs3znXh4QI-N-FD6CD(hCx&Fe6{ji@@jZ~tB`}mDv&eLa&Ria(-hA7z zv*^e_!ROGHb}gp(sAZa<496odWbR{=StrC`X_L^DiSjW2zMP16ZQ>EFxFcDSOB)I4 zPNR`Vn5&r9zY@#p8K;=IJA2lCB7Sx}6V)LCRjX!Zli`X_@vJ>Ey%{2ZC(j)*=2sC? z*)me}+5CDB^*y&;5{|902o+=H`^}$$p~-slmrgYie+zp%bj(to9)o2^VAk0df0o`_ z=V&jRZrXx(y2JXp=kjIxX_;=76|Dzmdu!9UPTLJ?Z8VgzebpdN3}FwvnNd6~ppTw=BmXH#u!Yi=l-6ONjA)h-0M=3UZcN zw-anVuAnnKF3`6uo@OoinQfSM7E~V2ef5R#toskN6SY zbK&V5)Chn&mIZ`HD)s8tkcG@Ie+}ujL9Ccc`S3s{Fxt^aJ%-cA2+lu};wh~;4lpFu zGF(LV)>+#~P zuiMheq4qAmFxOQd#r1oWKL*dhiJrW0Uq}GKZlyeGE#D6|W^;G|dgG$4e1BOu7Ro5Pn|5dgOR1;HtVyq zWHkcXX2VwK1e?S^Kwo-c0~M1H2=tGpd-A1B+;)&q(K_x362F!FHn7g?<=_V&o#Pu3HcUSmpQ?bDh!U05_R~?Lp!kC@U8%Q0#j4pnCdw zD;Kg(eMwj_j}wX)HGtT}RY(QuQ^G-m*!2w^XV4moh@=bK4TXn`eHpI3(893Gtxu3L zLq{tQ%19IbG%NH+`WZ)CrFISoe4P9;9y@V0y;YuPN5)fwBmSKkv>XkOPKAy81RGba zI#odmTv1M8uAnUXXI&^+ilQBs^Oqj3tmFv69kbm&+uBcRN*k%C%{_JvePior1?aM! zdY5pPhK5i4O#ya?voO)m2qASeRzY9=h#gdg@qziq&4ZkaWY|H9>SO4c+4xE9UiC&HnBjK9KxioYrHD8tlioY32CC z%SQ?G3F5Qs5dG^w#b9B-yrs^+cRqE$*RSL*Wa;1Ow=A#D3^T78m08?dUAE z?XSo&ST&$_n%NezH$b-WO-pCYInTD!^B@=Fhuw#!G-#SC?;HES9vWz8(J!u6tuECb z0KNXjiB=*_{KSj;u-FR#mi5i_3ytC{{Iz7kBEy#5AIKxs)aZMQ84faB(Y&MiW2wiL zu?$veuCbxL59oYlb4(T$cYH40gqLHQc(JOeE$*43OuUR?o8+UL_O8ThwuMIz4pKAn zTU3&nX2LPLxU0D*cX#dR7u$cG-hM!F&b*2AYwQA-ARP0?W)RX>cL^uSSce)Zkeu_Q zEY68OT+0U`pS*;JmtK^5S{@e3czUNL6*2hssNiNa^%26Un6H>lr9hgV6OL*auvht31YZRfV#gdpaLaTr|*(uEVM ztlh_W?4y>TwjJ^PRD@>V;iB{T0+ox0f<5X@#iL|5Xhz-gOV5Z`TeZx`-;X@iAs1P1 z&zIw&XSkEyYcC=lk|sp|8IJr1diqZwOkOrtW;V|E-i(~Qe9Y{u|2Gj#Jtqf%k+H3b z$$tT5zPnWYe*!VZD7;}NURbz^@G0Yc#^A9suW2_ zu$87TeYKIH{y$)AjOubSrk_jD)yb$h|#r zYvaL_Kj{7UlXKjb<+IQlw3laXIfi67_15U*5KC>7vUt_Q0|#@aw|CRZON-|XY>ita zsdwkf9_L_YU6Gh0q2oj)mR(Fh?F%?_S1p{Lh#tui!VU0S!d5(SAlKzXIfG z7FwTd?whx`Y|Jpplwx_P1q;1WGxh~Xm_*xIvUY4K9?us61-ls6ngzje2a#uo8Ac+3 z8`v_{T2k}+jbj=s(>yZ1Fx?CZ>mLa*aS~3v8i~CVZ%je$AC9!~v}G_>vgS#SRtn!( z->3K(V!fr-&}c4xtBYiTo2$OOVD&=pVFC2Qp$Fqi;vq3d%Ti@`EO|e=-5*Y}OB0d2 z9cnM{kfQB~Am)fK3~ijg-niX2pNP&D#?Wc2r4=ksH|O&ix_@$S8y+yB;_@LrX6o&UgZs(-eeNx{J+AtwWlv zd}UVJgR}4Ea#QIxp-FO?>8R7ofiOTjQzOBt4Ro}5M@o0Eq4^N*aJAZsxEB`ki?lXz zh|AecZy%O3G%j+4pY~qbw@7n3J#U})_X?V0vtO@XNptR9mmpb(t{P7eS`srvBNa*) z*!KjuJbpdz4uB3>*u8Eoc}R7B8^DgRT%eC`10>P&ZCNmKewDBEX9Pdw(dF+)YdrN^hCyZ~U zRhM`r*w#d&VhTW6LaoeYm=D0>3ip30u6dVrMmRdAaddFSpvV8j6&zkhY z7Vu{XUg&YPomUVd#dj!=;PE|puT`@sY{Vg9C10iay*7tQh^JQ(K^1avmZ64?CthL3OGw*4;r zq}_9+^tlJ1cnK+?)cK%8kkQF!Q4qjpEo!v=6i$Gq@N5Qb2=zZpdn06Q3Lw6;q;oUk zyT&#IB)_B57@J#DxZlgnNojM(KTht#!x!#kSFi6Qz2U zDcXX@ejTdVK*M35|4i?N)rCdn^jB6_U6I1^z)I~SAP(w)GR@tg%z%i>RuC@vHsKn)>px$k8714Q0OJo${gv=oF2kfG~lSH)nmK_8W-SD zBh|^tKZ`LXk7fp#uM_*F>y$K8lWyF#vJTAk)>fad*2c%LSzbxHAFd}iy1f^QP36Ba zFTaekVY43&xS?!H3KdVIUVJ;uhgo$^xNPu(>k7>Q*jAU~vW`PP=M4FN!#JuFU4TD; zZ!y*5c#PR?4HNRt_%6Lk{lJm}{c_x{*X4WdmHllLt2?&wxTad|Nf^%lBGhnKN9!CH z^Ig}@a^^NxKE^q(%JYClQ%l{e$aI%!xPptJG6|V3YZ}$p)&qWr2-ygOnH{XYM{8DB*y`PfX6mFb#HK^}?_U{h7|1eIA zj9sMnZ=_dy>sTb(!SE@ob!~B4LNS-xKst&tg+0nV2(QA3 z(ma_{A1ToI#o&{oPB!mp9cS6oumrcxwkweOe!pQmbH7WSaH}#HF|d}c`I=dLj8Jq* zz-A0BrZVP110o*IKtJPi@tfeS+QN?~fsV<-DpyknjZN}!`^XY;@gJT(V@adMWRZS5 zef}O2=R{v6&eMF;bv-+K;q2p)Yoje9B-)4ggS(gof?OP(81 zs%AL_`8hE95sPRbT2oPV>g%HA_5v;k2$)}vT+_ma%cRR0-5P~sm*f|7V)^bd5@luj z$x1$uy9qDi=#vQmBh!8*dB`B#8-MCj8$Bpc5N|~m-$wJ^`AuIz^8_iaakR7vPa7y9 z@pTppUp4vII{<4$(=uS*V`+k1Dtw(*nz27*Mi8WJvuv3nW6elF@SH_$tWMfVsy~tq z`-(s2=OAO;M)X@2ro`x&S|+YPpORu34nTNrydYW8XZam|5CPj#yeASppv3JW#2T(~ zq^gblCkHa1jbykl1Mzgc!>#0JJ8d{E)Oa{k5yVT0vAT+*zD6;%i*s(yjQsc!glWLw z$2$_tGufgpXii+sa#P>pvkZK6g4iDU&df|Tc&~{FG_+E#a{p+&%Mj5j#%yts`6oGe z_0wtLX8tBLj`PWUd;g}@?1-L23(R6umzD&E!cV_QpOo2XpQNYHa`pB6=xYrwD8C|1 zcbuw~5I#Jk<;fg!Vmp*p3_t4A2n1;s5RRK4U+hHTx&V^bFp|L|s9ohA8jrD-bNO%z z@dLp1}n84lN^ZoYv&GZ@9@WG)yktdzNwC+oU>|i-3v= z!d)FLa}A*g$dNyjGAuwCffTAu@}D{ae3Zho@oQofJ7E>TMchIpS+K|@t^XG8r*&^1 z;EfWQhEMm89Sq`9%;SBCcT`S1px=zEe=D^)F;p{r85Ja}{{=9@xwGAFQ&;uCrF#%G?6qtPx`l}m)Isju2P9KdOBxn`L znC~08v3c$wg11LRkP>fM(9Kz-?7?Y>EPUGvR8c=yCOFP0k4V5_4=kug!oJEYJ>tp! zLBb^Z`Wp{I;N*`+T11wXL?kmpYe(?&8hhQWtpLu}fPlmujI?a=hDDbXI1Nm!F#j$e zF@7ezdxn4I#;dj>+G?Ba^%Wm;yB8<#b&4l?Zm<|5f2=;s!pDCqs?lY8&};Iww!xNJ zJ4}ZDkQqq6%+hl&Fo%o4ge$Rad(oS+-8Q1O|1LQ}JVl6#ZG&X%b3<`nQ^G@Qv*P+v zTJSUWpmw;rP|xFKiCRVt3(59`<^paDYHG1-`NubBw(P^~Qg)bJaA*Y0!i%`w2A_a( z_?lO8T1Ir+$#G8Q;eFIfe~9zgtiLWN3QKgMHL^cNj4*Ctit#731+Osi0<#{p0ppOR zFw0<2b5+!*Z?x`*8vK(-4+nAjV|!5tFsFz^oMWjmi2if)B^O^O5FTNQaAPGfi|sCr z0-Y7)S!54hZ@L!9cGLk*kkpgqz&^b1Ih}aW5;zuV+Rr9(r)em%_q(q!-{5^`Icf(Z zssd|=>%b>>P;cz10(@v6>DAytumx!_-w5P5YJYuBCj^DjB0-7jR(k&HKOzZTkw%@$ zcnLZwll=Yth)0TRZeu;0v)lcCK9ZO3^?zHcN*hYM^Y`$2y3KwCf_rq`(^CN8bU@;V zr(rv9XfrtDQFYp0fQOGQW+J8MwalB__WUQ}*a=5|-{vKKi%*`BFy~WzJOsW#J29cQ zbO)Wf)O=#6{g&U5qyFvz>V84t!x31-VcA^Hv;J|qQwolRHr{u#Nl(*Cw8Op%nPSSI z(P5uA_@C$FgTetA#E;V3P`yiKAto4QJTUkYhpat$9VuG2(gVI5U1`Si;HXe}DFL;w za1eYTJH0Zyj;=vGZR6nxrfqumXQXUY0~J~iRVf;0&UBe52cO=%McSMfGA$kXCm87K zQZlo*c?$sKHXC)GdWPAkZ)+!6qkOhD+@<{e{lxe@)$$>-qbMCLN@Yu_E1lTc;}shtgGoWe`T{oy9eoKOZdmiCOW4tw z(>KyQ{0JQC5;s0geHkBNq8DJ7xTB@CceqL1si$(6*waz-=O8{?b>Pd^#M+by`%fk^I2MNTS^z3DQZ^CQ_<93f^aZVgiI6c8^h|F zeH(dX-^C<-4ojG>2|gFD3I>LdHJ(<~;5!ufgsMgOuUV-jQeIMXRKyVai0DHb5g}b% zpz@!=P~Z)XCzfp5aeAgFW>26_>B2e#Sganr&m5Be#G$-MF~*oi)bRD-S9jf~iOoq@ z6vr8TN^ntojr=HDFLg19&MZ1#$YSY=O6*5kKhb|%*T_)*fb#e1nGOgVQGL^sKIcr( z;V)Y%dS_RW#d5UJ>#nHAj=YWL`m;ETiZn}i)Qn7>*NmA@~`k6tH3aU=$>Eon=-IJbA7s(Jv~34?a*WPG~tN9zHaDM%u+h z&*pp${Z2l%mj1kZJgN@gR(Og~9-qYYq((W*O`aLO%aBKd=;okD)$Q+BX-+SOSBY7D zy3pH>`@qSTUPR#~)d(qF9>+^sR=?*XNWM!3Rd4h}#(3VIdOw^1FSW&QFg!_ct?FZ|wGaJvldqU4KAk)U6f&Kg{?)Oz;0<#{X~dyo{-xxr@br zIypAR{~~4ncf~}&#K`)ehVK8A|AQ+3&yj$MiRC}1#s7yNP>H-gQ1C;=>>#UrZzUz)HGzI1oH%!2iIWk*JS|51_@1LT3lP57|`2YLmh)! z0L}m$3Md0mQwM~Ghl9rg5uEOypV*ihUO?slMhgmyi~s4T`U?S&>HS@NWL+4U8p7v) zf9=CHyE-%jZ+88LzW2$g00RNmrV0dfq|hP|lonSL7g2!brz$N2NC4gh&W^SMn6e{l zQ!58_r%=xCf=U20xI2KY|Be9~TpHM$y~tbB{qt`S(E%6*Ko`%iSlMIAFQc4)z(?op zM?AMNKRJAS!!$Sns&`;(b^qZ8RtFDA>8Ys6+TaKI-JHSwB|SdBzGol2t-GU7Zw!vE zimqr%DWcxDS2cB<_}jm*vb_G3w?(h&vkM6R*ppxeS8nsy`doWMKdoC^S)bem-G_R1 z`6)|WpVw{-Wr_5i@nHGO=w_u2dHQG3*VWZ zdEocG6*%(;7x|f+o&P2^wE6Gty#i$b&KzQ-<3vVTyAF%vueW6+|`QbM{dxuA#-@Wcr{-d7n8_r-6A#F)HIsbiQzwC3I^iOeD zeRKMjzVUY{+&6gThyTluh~n}9^1;yH&;W$7nWp(q0;$i(V}s-O7asKo)bPEgIk3L6 zF^Qc2)^^jE@yGJV&-drb^ls09^jZpM_IDkXS=G?9vG?+s7v{fRYBxLUuln;g`e*ye zKlew+g>VkZ5~{85$Z!JPAU8MZP%|`(n}M{3eQpeh59vx{t3po&QSEpRD-pMTRPp_) zhdsH@GW4}^+w#5ja3qoX>QN0pm0HP;>X9P9FD|yEAJ8egF(K;yoikC7!g!=}6+ZWp z_)Ql=N8WV$@S7Y}X*j`+av(#V#XSAOEK88#El2?Nkwl!>XQ?HOV}>q;456iaaOXT( zymUrhS@5usaImXA{!8rBndexK;xoGbOW7xxsN2)a$4?gh&%{*dA@9 z!Q%oaj^vw&W(QPxww9F}mrWf?%ll}Q4ze-@;9pY`c%MMlb=zv6xKA})MmD=RLoUxS z_94bCKX=i8Z%+gQI!OB}g)=TK$%kQ@uYcnOr+K5|i+_?2K7RmS3A`)%Wy#k?G!SRh zH;Jh`2t-|W^2MriAe6;EkWUQ^el=lvEH534z>7+-b=Nn;%pZ1*)l2VoUSi2|ku%dq zcxA84Y66g)0nN9Th%n1&JQUNyRf`qv_tb;U64P&gDp!yO=uRQc2|P-{|D#JVvN5Ux z-3Uq~U#OxyVuL_d$XWNM2$G_Ky!=kL*saN#p5W($kg&=&5Awsny93E&jO6%xseRU! zbR&pc0Qq#mouz*2>a!|&1P9>Q2SrrkaC|) zklcr&!K`38Gx@K8#c`mZ-l99d4q>vZY&z}vN;5d;i-q7kVZN4v>6!{Iw$?Z9(`2^Og71rrdT7)03nOjsiGnnIk&Gv@3^rVwf2D4Xn2kR#pS_ zAw`~$1n0KkX`-@VBxI@BA6e@d5*?#RJRfm=r9pwgf5}MUJ^H5SA=q` z0=}KxW(}r@8Ml4AL8cQ$z@a(l4gsd?1fdr8@^CAUut`$$4|8%&M{&Ox=oGR+ZaIW z;p*zSp3=E_q+AYD`8)DyITwpA$Ia~8Vj7DFRxD^7?o*lH_D9P?49&kqyE85%xSHV* z*(xdIIWZ-!l+^*oen}3vqnDix{O+j^bBu9`GDFK<))XJ~Sd?lX0c&unSNfv*yfqGG zxi|Don4!9)(~bPoXgrO&P)E0+4DZKM_EY!n*N0F+9rR8IyoeS=M2;D_D z4bxW(^;;yHj!@itFqh)MvA{0bRN~h62+;uP`Uomm3DZ|2|8n9l)Hwppr)L=I@Z~W{AEj%jG`R-1Hg085)y$m{igx+J3 zv+o#jxhEp292o|+O^UD;r&BcAc^OJ#h}B#pirV)Ik{sSg)z*^i$m*dkXsaVLA%|f@ zlIqO;av`pWoIsW>t+8L3-`TlSTrjsc%^jxe{m_0GM>pO_ru#F>*yvtta#^9cb*A`} z8HMZAglCHY6jmc&DFeiD(>>{zJNNG@3hqrR*@kO5R*DKk^ak@J{{{OeR%9O)K-Qi2 z%pL-lb31YJ_>lr=l}wFls^SdfMTAxv@s?~`t;W2iO;z&ptNP?jurCZEMr|nRs(xJk z1wIJ#c4a1=RUE0W>tN5-7J#v4J+oTCCx+}YQfBFjFtl@?15zl5x8=9xmJ+Wu+13*B z*|_pt_0(F3-k;8ib_H`f9OZARaz)9Z7K~aUz-pd0EckD$c6k{&l9Tkt*}STy;!88H zDRX6Jzs;vmhgj)KF(1_P%Gm!&e3<#zC!yYi;)5zjs?&#lrO61T)R6E>h^k2piq_M5 zb;XZ(@%t`FN6Tt0_g~D7EkAucFNH)WZ<>=ZSs8QFjxa;fv%&)3+7bgoEE?N{Fl+C3 zOrT0m3jS||&z(B`{=UVgqD9EarlJblvpJ@St#I)S?(&3Jo`oL2r!mY7Q>jIyENb^d zs9^6LSnoOL4KLJutK|v?C+xC_w1AAZeKC_F^-Xocsepq3K<(R~&k*lM&<_pg@By=! z9D9f*)^bWu-rMhUcI|dJs?9#rd4!24ZZEKh-B0ogUKKu+%^3pjl`u?O9W<9E`yt4D=B; zXXOrG%Ntx$`{PqMh$3rn55XTHIsil4L*ZdKE?fea3+PdW=(4q>3$0qVGUL8nci%5B zFuATUMEUL|j>V7_AYQb42VKz!v%0DL=7PE1b;HH0yaDu;L?UOWkj=5Py=r&wN6j_z z6ZJy%ICs!?gkwIYj~4z>@&qI#EC&Dg!qK=aGEBd_K?*Y0F*G^P<=r1_RvS7OXr7TR z?|)*+O(VgK<#7@)k1n3Jo;`W@zf?MbGn_7E!1~tZr9TFI_g}bzt|cteFM8YRvDL7D z*&*j=DrX^;FXMGTSg=E$z$h&qtSnhNvOr?>&V4o>_%b0F9*@ad?eOZ$p;9Y6U_9FQz)$X&3z8?r%^)B}Rcaam5vR>eGr zH6euJa9eIFL4xX2F!e^NgoohM{BaRbF2^*HHwYqqZP2^b&S~{Ylj$Eeom>t${thFM z(g%)>DZdp!#4kiL2D;f~6%l;1u2rnr_VAFvq+2Q(=IKnIszc}>hjCWLecA44|7vZY zaio)_IEMx@a9s0);p9I6)CJuDKe>+_t>OdQZSX;HK@Cn^d2Wo6$xS;2K!o&lrRM>D z$F}7ID-}V!4l=bPxI1x>)rk))IE@7wwi%3`*w2#2FrQR523ivuEMHB@jA9#z)IU@V z5W{U=Ao<1^Ucw$aY0on7-0$vU-JqjA{w7$}SYXY+B_33LezBc5Qz9yTO`{w*)lqyB zMR0#7OKI0r06#SZLD>kgdK(+DV&bp4nQb4v7JYY(Fb0ivjduft1RRV&E2`J(?ns;L zCix5WQR&?acT9ppXxyai>m?wuOrvK3v5F_&SX){#ADr-~)ZNzl2yUVu;g~(O_8zVu zWdO{4s}zK$k<}pq9qLf4X;Vn_AJyIeTapsvsj3)bIj4fzT_7Zg^iWhJ_EpM;u1}V8 zjF#*6M*J!GoVfFS28P{u8x)z%y|tSA8oyQ6PvMRnU2iU3*ijtOnry_H~n9p88|VE#t|OLDb=`XIrv+fZIstBUoJKD zkE|=|ICXSiqz;wzMaIQfa;7CHR^gQ-OiZN4v`Z*#wtm@NjyA(017jOE+hfuDKgPMl zQKIMb>*eSe$+3geMPyJjZHJ0>=g7euIgIQZ#zE=Jtigw%wu}~kNt=8W#Ui^6^cn}l zXg})pmD&__)XOZwtuRQRr>Jn@tnQSNAGyn!%E>dJ4rkxyN%;FD0MY%{bVW=#4H2mieY#sc{$i`szXOLi6Dz{(hD;oni* zelvK;Q1lw8FPS-WP%Z1tP^m0ypE}hDuo`45SzAoYrs zt8-~~R}*&4n-cCxxnMWacwDa0=BveG8@j)-HdwM%ypmQD8pa;xNq7x zSkr-M5zhtbLWHA@T2ekcR$qF3^Kf8L7ImKGK&%o1H+*fHJ_{V*T`#RMP>kFJ9&`+n zz3G~UgwV;>Ct}9>furJho8yjh^HOeY{?p=?aX0+6{U;W1f1|<&Ou13yUw?Fq&9X8@oc>oXu6^OI*k= zMoLy(86}cwpHLUaEBPIo?50e+xAr~@{jj^U9sC(+6rdydsx%pDn2(IP?b~a;s1|}P zmnTdzhKvAr=xfYYR_L-^!V43Bd-O*1GxXQ>-nKBmdyqR3LGeK61?>+fhrx-UN!yMx zd$dj=b`xf1(|WTf8oG`+40BnrL@pM^-NAJl+gDe@@6M7r9r9Y&)h(FkWgN}O%Ai4J z`cMnDCFu36*7^m{@bWwJAGxBU+u)F*jzLMtp4u!l)D25nQoK(KVYstWy8v_H zJz+llj#{Qz zt?^Lzb`VyNGBGfwNixZ*Tm(zGv~U;kwDR6)Brc_BqM?G!;?~e2>(T8u>}BYPkHvV( z=4xX&F+?-Jwi_Rr4`UJ&JOP-xC+VVjJDr^bz|U9?(uGGz`)`d*JGeGtMA^Uf&SEDs z<*>0HJS6GKCr1ix^L@74+=F%|t!^RhvqNzU=@q$rQ3S{XH(wwW+0@Xc`vBsrI+=X7 zM$fI7DK{SNy{YaPTAp%}#nH>V4jV_WSMlf{$Glc!X{j5k@J`H+g(rqO<2;xKDNW&y zR8B*&*o=8oG=W`hsADfZw`?)r&Tn8O)Qo4?9cNvjRX?r3OfjvMmUAksIpF1mrw!wX z1kAqWP?UGvZh{2AbQ~peD8{lyp-|W$K>=P8nU91_Y}(J4ul)Xww}iG$Ia`fVt(3c; zxpaYqcTh$+yk@1kgX8MPwJdps#@zkR3}B6js|E*{EK4m}rXi z2EvmtirxrY%>g#k5u(&@r_>IPOuY3gqb6vO6N3^L0Mbiv0rR|r_s(3z$!fPt-&Dh+ z+I?@m$mk3Ar-YCyJd#1=*(FIw>;|vJM zn}x9|ZunI&c&P8WS)=!|FO81BfkJ||Rm7!+3vHV?rQuo9=twe`|H4n^Mda$!{(^2q0{*sc*r`+#{N#r1~W-e7CpOn0rlPlHZgLK6g-qAfBUZ4*d6 z&ZY`}b;d1453t3_oE`@niW8H5G;2N}ZciLds!12n?rs|CF7oo1>c-#W?b3ohF_;Rb zZIT029#`1=AiJz-^Dpg7XGkBs{yXT>hQ98W@N8@srysy0YRTB~z{;OS$5n1Bw z&n!(2atGUHu7qk>Fz>G&n@t@M6ov;FeS$Rcg`6Sb9+15|(uxtM!{~ej;9_4d0%eU< zeNFdnc?kgGfD*+T+VZ95}=k}(4yEt@_OjB?y$itO&c*{E8j8DM5(YRy??eO zHXw9%=uaR;UZhQ{9fGU!i3~QtVZ>G9tQ#8(mad#Q!#?V~ACYf5yg9jROf^-t$g3Fs z*v#RT7AT!Z&#J?rvYD<;56&U~6K5i-zH6k!TUW?t{5l+^bF}fUltZUeZlJ!FJOX*0 z9$_7bCh}vb18mI?kJ*Rzjv6FtFxRk(j})=Y z!hXI`18R#79o9Z(!zJ3if9v)eTh}?Ks0~~n zn6$?1i^9QWy**_J8^ohzo7a`m8{ey75z?{Aig)#q9Raphp>wT&EL-(1LY&jdN|&wD zc5c5qocPo|y>BjOkX~FVx;s;&XD!7!7`NXDPYN1wdgFFTJD3xmr&!jgL6ILAaHUhl zvpD6=PRz0*stg^I=kN=K*S31c@dKTfh*xQ6DjR?NC&=M?yAiY|O4UvpTHu@!@p{o6 z6`K?j0W3ZC-s`CA->*eJ;PAmaG?|bHAc%0qYel`yhqPI7K@)=1#-*Fq>0f&HiIV;o zh=9Kbk$Z1X3&FSun92cz-psN*0;j*yyMC|5@M}OQHXA4cUyBk>o;q##3R>gxwXEDo za9S4)D_Z_zn)0sA3$J81Y)qCX7{dA8)oUJp2$c%9p*%n%wXwX}^)KLy10Hf?JHWcv! zZ;`a^Ng@QiY^R&5yX^W7#{yd~L?~#`QSGzxP!SIVw_{+>dn4`IRNoF|d*SRdcIcxx z8%atxnyPRK5F3Hy-*C0lse$#bB%`rKhOs>w!r@<}BkC&@^L52ZVauF`BqlmIL&Mjg zwX>bHy_0bEYaI>aBJtI6{ZhFkDr%LmAVBU1B@RX&WR~`uP8Tw^y*w^tT+PKd<~@ee z2N#QKz3qo^{zI)sLwp~w84H%}i0^O-M9q?83pq+TJBr-$!!B((wo-)Gjw+07K9 zr}cFy?lu)QrC=097(t-y2``=Mfo5PQWJ~pW=iq?3xr)Pm(L|rD-pQ- zjp4i$;+I@I+kv)%Z_CSdw(K^mDx4xbyGZ&Mt8`?kiX+l`90J@nOo@T>OhV z|J6l}ecM94Vq*65^w?Ux!9*m5A z3jMj-U{ETT%<>rAiP`^#qrMK{8E#m_A>a22%7Jo2W{ zHbtQq#Q!UfjKu|`RTn0 zulnAZM;&Qs?{-tCTDD*teV zfm0|KDb@TQf0#t=c6vd+=we6xO~pmUsIL$X+Phk__Dsus+jh#}AXPJ@VJTpH!dS~@ z3pbo&q+rhzV)I;VWbQ+!wMU@FGN2b422q2>`sqP3f=A*zgGi$~EHfdSSjDgGG zLoGB>k1Ts*p?odC6#UWyYWL@0jtDfBM46GZOVN?0NTa|$ZF;dRia4qr;x1JFypXxc z^kK4IfPNqy0$#oVWNw}m+0YW$CO9L9r~}mv)9pJGOZ!13BA1WI<`5Q(Mup40W?V}FqbfCtCjmn+XW$WUx8du!7)$y|5>L-sMv?@K5gkMDg z1*L0aZD$i(xP32|Kq_Fw_?b8~NV`$S9cS z41PMHW1BwsIdt(i%=ucPk)8Y0BAc6g373$VnXEDQJ7F!xYbjE*Q9&n4mr;Jd9seZN zNP_b>KL9xj@0U)H@mobB3l?>@+j2TlEn%Hed<;Ug?GMb$iW@w4 zbx5@3!m6ARCxunHFlB2}Jn(tx{uQ`U*E;Nx&|*$Ub#RPOtF6^%08#GrFb>6_(fl0d zS3mn+QfPbyGDAp$Q%+Eeg~k>{Rpi3aux+q8~HW(JAG z0=q+30Kye~$Uj)!j~R}KZFMtzmcBT#bjhvr_(jUe3=tgX4G1DXHO6Fmgdi2oTiC2W zE6A9;_|#}$tejMh44@pNjdjAN>Uk3f^&_%`Rsph*NkOfCo)8|<_o@>pyxBR$mU5&A zw&}yQjg5TI^ZoyQt^oJhs3W3u_U*36KKNnh97Pdpjcgx7QwCg$40nm* zVf}%jD@_L^)(q?c}rt=Vo3{F^NxnX?(#oaVsY5i)Kc1c)B(xYZ+2F(hqi>gyGv_I zzU${fAF7U3tX&-$Eci?~I@K>)+K7zVAvuB=A?QHdZsxf5Z&BJ8qKuNb6WHF(8;@JK z`+C+mhfKVdH+O|3vi|UH&oE6Pz9Jetz&6wF4U;N zQjCavsYvnUk}0jyPH1cg3KUrz#|AV!3|P>RnI9Ofg+rmvhY2D5^Zaf)lUAZdPEsUR zeyyEf&14BIz1rHBT7@VD1-MTU7pU5s4sIU%@lv?YI_$}VH_1n@A&!cMNN^|93X}PB z&3!j;%)Jf}eqp`&djSp?%qf6ReX049p=-ovawWUJB{B-B3gV_u#*&nvXXsLWr=cz9 zyiRG->X_JT)gICSjXPJLT5asZl!!q~je8vVp5CN_?3odz!>U=1YlAd_^D^^*{iRH< zD8V2%X*ov1BSMI6ZUIuP0Mq3DPextNK2>^kenXSV*Z^gY6WY2-x&CtDA4u_QRBkD~ zkG*V= zwDC+*Z|p&Hcr8d@SD0gy2*2721kccMR2Q-*Slc<>TDjK%N zA=n*^^=A~?N1^fxfk&>^1uVuA*u}+3KEeOUfA%)l0A3q>_Vx!j0R!`bgt5a=%yNo$=K^D$hx)mmTNH=ULOD zJjmO6lIG8YP1l7kZ79*|YPo?yaIh_r)XgB`#!(VeWBIr7SgbHntx1>%4wT@L^gIA*|)$9EyWxW zd85O^Mu|E#Zh9}luGh-B?6Ra0?`bs+A~>sOw*E{hEt^3WvfPOZNQ>YFeFMJ%l_U7U zvu7GLjLF;J`wq-YsPB0i6Ck>WO0))>Q^EvbKAPWvn6H+#a?V5^X!Xnu>QRRGL2@G@ z6OSKgShQmS13RIsCQl>janJRDR9Qm|w~s(YF;uRnZiRp*iHt|G(yBCRa1dG8Yt5q! zrYHr~HH^Ev>kmFY8IT3_yVH~=ipA7bmQ64y zP~H?dFFZCOzbI4PJ@6UEJtE{+IbH8ic-ijPx!&CzqS%b-nG3>|?p0kNTKMc9Vpl=w z6JN2)8;@z#ydt3YjEO-(NEro;_ul`qX}c=Ze3F=Fvz>usb#Gdj2djyJ>0Nmo%3KT6 zEC(sHqBGM3j>=7hHifknhlObBOum_yRh2G7jqDgGEzrU4;;|Ymu0n)wbDNe*Z_Xbv zYUcpTQGyZobm2xCt6F|LJ4n$mE0uF}zGbxflCoajK$NV_Tt5s~I&npb#zWi|v%g;E9&+$-ry+e0}?pYU6h$EL}H<#1oqlH}Js14Hk46G0NH z4!7303&)lv=0DbtN82xqQTZkN3o_FHkkEnkIhR61o&L=}x4jct18%cCjDbQ8N3x1P z&HIK&f1J+B7ttE>3+(DITK=eMEJI6-21i}w@N|5Z2<$@I6RwVVIq5(;h7^P{Dva|% z%z)hhk9wD2n$k4nmyQ-sMd2RCky={g$8|PR57W#2xY=qir5A=C@hux{aR{Y!ft557 z-j<)9VH;FGhd0 zZ}qk7BVC0J=$oal75r)Zaj`#jwM~~zNpOBr%94Cx5-9-1nOfPjxRkrN=uixWicUHU zd$sMv11!DisBc3!5woz_61b6zJH@z}9{AsLv`58HNtJ4G2jyU<(dE&P>;wIH#i=P5 zu=RnlC(WSKfhd46r|YL(wl+2LS@OLf(#gAJi=Q^|$r7e%;Q@9n+hpLqk)JE);}mOT zM>l(kBSPzA4Ejf^duBso*qG~FF}Yz&R8e;LhB|k*8$wZNPx~-C=)1?q$}m z?2gJO7g$!c> z+ZgWQ3Zd5!UNtmEzUnlq_+qGY2(}(UINq33mj>|yHDagiBKO}6#u!S`=nfE~UIw>N z!f(H!o{AyWi>vKRKt$&Z;j=MgmE54kai1yamf{I>0pG&3SYxrG@btruKcq%jA8RE> zC5-D+-cGsMim7N}4ERA%yan`{)6?&5DcjpfkqMbA8uUe~&@!yu&9vi-z%i-`t?>%p z;STw%CYLB3JZ!M$(kzFF9hieSRajtYK{;#EamJy&3faereGDCGyI`e@zY50d%((vq zxbA6z^Yv2S7F6p%$-yNnTm`&;uEDhSBoxdi-#sjfNMX+I z4wfQzNWmi)>RQCR)+N*R#?^1D$R!^M?NG5`0Ht2Ef?vvOmHX65hA4EX`34=87 z7;&PrPAV(ZuCUmVmO%y3Sk_#}WWc5LDA2o7zeSJz!!d)*9AC~ZGtcBabWnBuZ8Utm z_yw#S_*RnpoJx?!&hs#o&UQP47e%f~1I6)C#a{>YI8g1#OQ(LIVvB69EhjP+duE{p z#Dg_)dXMadD`MtHvZMS0K1p0|CNDGkx|J1Fn1$y9zVf7*F!G-F333TWJvS(&sPZYZ ztII)yqDY%5mU#u{&bn>B)n~4(vWR2jMnzU~MoNTZP{%*ViE8o)e7xp7ujjs()n=7b z#rp>2LBT+BdkuahU0VRw)cZHB+0tDs8!agHbeehH*_S`W4tkmj!deLggo&tGhp7Lp zExI+}lN{@iop2WtJ1JO^(%e+~Pf9^I^(VH9Zg=*!?CSH$KLMBemdSISDcA=-|AO;E zfco<_!H%oaaKU{FLsf2=8Y@1u&48a5T-WBh<#*pLl*$7imN5#Ne?Y>^y*>8ThFmQ3 z;ZSrmB3R&ETo=w-pIHsVuNlQz4X|A@lF{pHEV0(D9*#)Hs%u?H@u78ayWn5Kh4s^r z)pEITYB85snYZej3sD@X>|DBW1#}pEeJ7aX4|t67hW{>khzwU^o#w=nhTh)l1+Rp9 zvxz!;Tre}TFpgN3!@d&2+-qlQi@d=+<>~ zuCdz4fvuhEYTO1g#EA4)E$gDJUU!>pam(JM?$0x$$2n*3;}v*qK6-&jB#ru}J%Rj3 z%HZdl98cQfI?qMAN6;&kj1}~FXvEP zR*w|N1#Igt*yOk`!Yv8G?43j#;*=@=bO}q_6m9&Kc$PQXN$Clo6t`$`m@D=Q^GL!n zf*ogMqi>zinH z#z}hhd4=Jg1kf(jp9_{)F>A-LOChd~4Z#U(AR2#h4voM*1NG z#^M7Sk-l8PnyY;7&iZ?gU0R6-+_MtzgxeW4M6rN|B#ksgNq4_U3cVVz0E&+n`55gx zf}oM9oxdm>3xM2KAo&*I1RGoZIP!At>1`prFY<iCUdiK;wWWHZh>7hC59d6rh`I9fb>nOniALkM)c^p78;r>n~nybIvzG7s<`RSz~aEsGCK-<<=!PH)w z(q4*xK-5FTza_tl_KanKs&?LhpC^S#@V2yUVtMm^^1NcB&!lNZnZI(Z@TMy))2O=F|@i``qO9cwkn@LY}i~I$du<(Nch{c ziE=kVBfg_@7p{L4GzK8cv#aP5~65T7OWrzkF0fTtj%h2?EsDJO1}f%vTOD) zQrM6uNs=0%s#mZ&?is5Zp*3dbY=yd~IGh%{)!j+4#o<#ScEN8nCK zat9udXrehXjgn#VRr5TO`-Xk*x)u28$-#Q9gCRMdkMEuWRvp(6N|?+W=Ie;(G$8FA zr6Eg=)wEP`vR1!{E$lak3vACPgYU|6UQ?u^H(l8y6sQsi^L02g+{Yl;Mx{->O7bMk z?QC9DSP&+gB;MjWa?AKVK?`}fn@wjuPw7J#{Ume0a@O;YOe)tTNA7RKMYHT=M&c92 zii!b%pFw3(GRf_i=o@D?Y3IJ`1TP92z#LJ)M!!88Sn*)iHB|lZ0~X=2g>zkoO{RLM_YS-* zvx?|bOHQ%O1_6WDtZDiSQ{8+J!C>fFU-$35VFg{QQ&U6wEwhb{$*UpSeP-T>8Uij#J^Z0!8!8 zht@J<5LL8D90!&nQDe%`N~hp4Lh9@vOnrURK=23IW~yIIHU!?ckk-5lsnEz|{=@q) zA=5)Rr8#UofcL(DsubRutLLV4SagMj_x90-L>pP;iexcC;b%f3j#qL78VP)B)dulP z;H%%uy4+Mqv!Q!3I|v-!ZOtPIal`~JzZ?W5h^LU+(b849GeJ}i->Ws;l($J@WD1SI zJ+T#;Xy|19q6Iij?(I07+mgy;h-0N;y%y4O7Rq9SSS9ri1w3vx?DH1fgH*yaE=)9{ z?@ji@^gpk#8PJd+lOOZo3hrUK;*#Asvkw`L9eR9AM|EJDCSCj&=+wx0_OLxKS%kn$ zBDdWC^_{i2l z6}0bX)MzJF0LgW#y;+O6%>w!CKl#M^>u_-buU-B!yu-DKj{KEt&cW2qeh33Br_?8i zT^!I5YWH7OBN!*O>~Ln30o{UfzPfg+WPZX3hOuJnWAo|u^f0Y+RkP8P`MQitNgZz& z_6^WB73>Hh_?eZ+A)lpOT3%{cJcX$&^H# zwJXu}IgCg!eM~&Bkuw5UUi7P^m0E5iaVGdOj}w+i#=fR2?80s00WVl%Yl?8Ms~#^n zHduqO=P=TXyy2$*Lg4q;m^{^KR6g?uxo8==Lf_4NXPQ;%PWPtWG_9DCrX)As?95q8 z4_||2R|q~ojRxWns+-m&I*ou&y)o4USJ*ZeP8l`}N?vq7k!K;@~|{iTXMWHT|RI8k@uH5#kgc&@J=;@V&~{R14h3{-!H@_Wg$ zOr2UfwstU6CG3NxKUiaDXG3Y_>W*&1In(S#rN}KTI+9~VxL0=@k4k^{vSDZOY^I!^ zjhKdvYQeS-aBt6(m;5{|2Oh3q_p#^}Isgw}s~y{ZxmMDAZ2Dyc?C`!jSMB(nUmWcJ zV(cD*1PK}}0Jm*p+O}=moVIP-wr$(CZQHhuo&T_h-Pps{rJ`MTNYRM z$DgdH?EPONGq`pOK-2x4oWBxE`J9m`gr{9z?lga_)9pKf7?m6Z+#z8qXe`IYtom~I z@@fXd(twE=WefsD+d)DS`yRsKpJRX9wVuql8S_g7N>)3=29kpK_k0lUX=c+q*pQvW z+k%Anr8=`vceqsEu0lxlCcgYgMc0^t&ly>~$j5dHxEyq20CP8xk1VdIx|RY~7LvNvgvTHqP*#Z7Usio76bM-7op9gwN9oDUVGG z(o6R!Oa`bIfflOm05zo^3_Smo;rB%5k8`ZQwvkq!#vj?{e=|s#gFX18nMO{mvJ?5z zOWY+FH4HXGEH`gJfU5#@&F_o1dpDRd1s8T(wk(zG_>*D;Lt9;JLI>K>u=9!v*+}U= z@`So(NLI;q!rl@gQ--NKEslsDJMr$NS+E58WQpAGa{~Rz8?n2SE8g68s_uXQzT61z zyxsNru-aWIWR?`|bDrkfR{kYqE;i~sDfF??`cX$i78==alh6VZm_-z!q(dR(r;>J} zqK3PaJpoDnLnT@dzz|Ogb*5X6Y$XPu#B-3sHGvvbzJoceci*%U@eCR3WbCJNepMfp z%z6{)jMD|VUx~ZB#Z@02@EXHp<38WCrjO>Z`8JLc=gK1rdCkY0gjI-(Jo>SiZ=F}Q zn~80-?vwayx~c&hk*HEFhx@`RZb0N{p*SG=7OG45BOaYVf&Fx;tsGd&gx}tS&IBZ) zn_cfq1D0q>!7FjnI!D$iIxF@ESRU%T(~=)SPC_Y9HgC&lLMrJQsFx_^tzs7mDp^Lz z^8}XG5htwfZLzdo#Ts6|S_*Y^NZAz&`&ygT3rpr>>!r44 zKKYR3HdVeX+E6NxB4*(WZ2uD!{_(y(j+N460P$sIByWSB-gVFBOhZ7w5=MXs1>afv z%d~ZU0j-{(wZ3n_FK z`ci3PywfB*>wIrq!`mN|4$9P>beK%w$D&ssbf2qR3VPry%aKhk^!`XQ5Fs18P*q>N zEl7zE{9S+t57HBe(aa~nr{*u$LFhiCVe22s;n}cvw}_IQ`yaLOhJtoEp9o95E9{)c zb7%hT$9{ZM=kho~NwASi9 zfe6O%yxhyFT%MgvrzwyRvhI-}Hnd`={rqf|N)2Mlx^3;<%Cr>+#$bL_$#e82T-l?V z(;k1)Vnk$G!LX2<*?-wjV;BpVyYx)#x%y8CpXs2h$FR+aSPlgdpk>~(N&o{ zb<1#OgTn$vNWQOk9K0ouUNk61JMA}fl+Ic1B-v=86_wlb95hB-vym68qTnE~yX%eB z-fEnZG}`wJ0?G9_`acun8ll|`CfWe*tv8R#(b%bgdXzrw2?j$iwPT_;vd?Q*Y8tl4 z6l975Dj=04!!+uyMfr!$~?0X<(Xqlv)c1$7|U&cC8+9 z+TwX(y5L1(QqKYjTSRwCZ)xz=P8oUA$00UWHzUc=Bo(?N&dZ*Da;z)K9en0^FK|KeZxWZCikg0cuIZO4ZjIJws+%4%?flrC-+y(k$u;UB8Q}Ze%|z3lJiXHzF*YPDLNIJG5A*Ow)v9Z|+Xg*} z-?;N#m+2^l9ln)gK|@|?3ZKuW!Xr7bHz(`2MFcCi#k*bE;{#KC)F{2moKAH| z=_Nrr!b1)VXfGt|GJ*uPaxWF}tEv2x?u_CXUnp{C7D%PDzom=+bz@o!v{2w#4sb;R+1$7($3?g~g82d5tN=6*=s;12Z^Q!V`Hnb&ZG-BjP3GsLJ zFdujqe5RrJ>n73ev(QbsCXtTsrdH{TGl>-##%Hb>Trki$;vz^WA8={v){ItImY{h{ zHCx8-`2|WI?5Kld_ClZn^9b4xk2N_#s#X;$PvZ5d=^cK9&^r5o&p=B1is@B%yi9PH z7sWiNWS7LfXzZvYPiHj!2o2UscaWabxcUE8ZXhUjmsi~p5*e{Sh zZJkq^uI6+9tw@aZUdoewR?{DS{QYy}E{5KkOJ)`)3sd6;$kG&mI~;odfSLfupsl)8 zkO=Z`wX8S8!Gy=#&(;<1RNT$oo4C%s8qEex?Drt!OY(3PGdVjvR(64DLg`s94vppa zLKc3cx#EB+zV!qP-mf53MKwqGNsRH|#L8W@&zbu;Bl`W=RQy%zwE{Yboc~ViLAn3I zJEV!oFs`LNGBO+x2A6^r7o!LeL^$_5UIK8Od=+VN%0ub41+ERS98v6{-UcjbFd#H;X7G=R{)eac@GOfZUSso09Vk?%8*Cpp?e&~xFxtb&L8Qc zZ8U_er*T;Q8mE)s=&>=EJrmO}Mb1f!A#r!}g!q(}bM0wkwD(VH1kdHp`@om?ew>$; zwtZS@=(+GDNE^x*JB^6z};257{v3EEEY)sG5Jqf_>u6zJ(f2@-s3&D1uAN1NV%a8Sc;>JGkY#$P#o2heB+W z|K1wD$TnQ&9+7mT$Zo+zF*0(A7emStct$>Nx!O7iQy(82SNP9+#4-0MvAIruJo%IW zz+x27?_Jf_o3wv7k8`VqWwd}gAp+*Bl(y+q+g9gsK-MBkg|84HJ56a=Lco$Cdr96L z37dPOWNbnG+?}BFDh1iFmJoEZ0tX;_PZJ8dPWt=qn5ui9iwM{G%}8&e5k_P4jxA>g zP5pYAv`M;ve&i-oT&8J+8$ueI33(rxvB7mH$>Si5^xBQE-0E<&v&qteMJ74I5{3!4 zabI0e5fJ5N{Vr?OQi@efd6fFHOFolqqYukaeWQs&OWXQ`?I!rAp;iA@2%((*8eS>< z{R&MVNoP6-RNTus$}WpKWbu<5GteOE33FUe8jF;2=gfd=O5+vQ_fqm-_)bj|3Kc}T zFAHwgqS1tTGbe{Zz99M6KuDaFOX&yJ-en!%M7m;Oj_U@6M2jMxj#0QIxPD+zz_w(P z?!(ceYmUPV@=dweSz5aK2~_kdRPVTyC!2u9SvJyN|AwN%Oq>aG3X1^B0tU`1YBU64 z#6WY*NQ@RU>PAy~9Q-7HoUceaqK*iWCSTODA{TT}sg&jxlA%GNotZi>Tr7=+t$lM7 z?-3q2C21Ssau&L zPkTBKq!CwjV+gCH$!Mf*#jcnvD4Q7ogp0H`px>!ozDZnB zp}A~0c`9qf{q`LC(cUq9;sN4VC`X^;u~NDzBR9DKU7CbM(hShb&AVVZWyGmy<-BB0 zU^Bo>#PKe0XyGr(+Mg_%`PG3FJs8A3OIhDf^lz+M2*x915wu8`g7y6d4;o-g3xu=^ z;CUz22VdDidC#j$jYnaHC1liQrzq~~6T(%6GFtrl!Gyo#g@jI5{Jnk^G-+htc`hK$ z0(?PghnXg*0!lBhbhDlSkRa1>n3z6mz?lpY-)-Hu#;|sv_^R7mNM(Oevj@=fd@kH6 z0MK4$LK)bx#B|d9&01|_MeJ!<8@;{Gcsbs+ndrFV#s1rOkALL`WMn-qq4o6}xqyw%Q}kkT?$ z>CMDYF_=e?E5WI;X^s3Uc2-a!4z&ibY%!wjZi^jM&{ps0ym0F-N_O{Ai;UGt$5V7( zNwadLN!h8eI{z?ey$G%b(-TH{uYGVr++E1SzPD7y*KG<22MrC%12a|*XXL*Z05JcW ziO(0#o8lo-s++kVa5<0WQRRf?;SlPcgf@7@S>sc?^^?8O4>Ec@e8r&|+QB}uaqB-5 z^>KD-@tpgc&-I_7EFX8TEzyOl^BeR;&)hB4v9)jnC`ccHgoI~e7VF3(5CemvhZDj- z2r+&&)3n<;ho5CKnL~?9@!t5)j9DZ|Plm)Zg!)6vrAxaU)xpQ3we{|EQsD{y!EC## zH?enTSv=4bzM9h~o0prjUc-;)KjSW8(<6yXI>Q%0zD_I6$zg2Zv&r2(>Y$a$jtrd9 zm~A=%D`=mYPcinBR=<}bM~b570tcu-5(|Gmz1f(1XFmSu3-XjKoD*k{P_$HqwGja*#54j*jF52#JL0@(cMXRYrEmBSc zpHK&Hg!`A!Xwc9{O-+O#D$SAS8$PzJU||pV4y$=gP)khKnd#=+bi@;{d>Apsd`kmT zmW=>v8r@T4blNFpIBN9dGXh6w(_Xggga^@GQMRfc^Jr4Z96x; z=&q17)157^hCq=L{uj3MCOZZ?nRPlyrzgC}A~rUh5~njY8oq4lAtg57mZ?BQB{ikr zI7%Gasjs+6-J+6z+arXEoSL9nfGjEX?N7uQG7s$jN9Ll zp~02bl=eyYIfik*ff!VOeU#)Jo7H4PwO$|L4e0hen~rsgVPsg}%}u_7yI|^Eh>7r6 z&5=d`=#ik`2K$i*T8$?A;D^Iz>D!)5z@ZJ)ld^ZH_)SdmY##UOaiqn-dNJ`23M-17ReXwYLs7!KUcJ!|g=ZvIp4@0SFfH={z+&nat z6c@2(k8Lnj*594+#NAVC!pOLL(vYt~vr{jQqlLTu*|?4zU_NuW2%zM5zjU!)Z!p04 zam7W>d#Z`YvoxCfs;y@Tp?jdeUBd@f_<*J#WnV!y(HkTMA>HRYNkIPYe-Ol?TZ;^- z{qMaf3gKlJ+lY`y+ua0LvdFSteb4zPwW?15*WIH~gX0ttqS+GiG? z#Lq$vFM}O)S^?R9EwZpDw(Mc+o*Gb7a{_Yb9MDCz;H3mYAP`bc9JG>CN z1PPF>b@Sx_H~umj*KEiApaVDB3US5w4_zexB-Kxwd;YCnCJ;nt<;z%sz)g)$?bY!J z4qYzk@aVsc`ys$3c1RgF$8&f&AF*@sc0a+qB6+nMkCOa=zRlK`ds9Q^)?}#9hEd0= z$Wu=8?_z%1J}@~2YDbGT9OJ6_R9H`g!8WqU>IGRE-(P#FT630)cUt58iIT2Bk&CM3 z2<*KNn4WaY-?Xno-sW zo#C%Fmu%;g7JX>%R;_51d2nS?{DL|GTbr}DqZW!6y{LV%GEMu=2`VSgQkFz%)F(e2 z&+HX%Gbc{Wp8E@6WYt%rt}O*uF38@FRL`(7#oP2b56z2({0=&h4(qZen_vOH2`Fjl z0g<*a1rfdwNDtEYne({nI(vae&p)ybkC&JL`f5GW^14mn%-L!D%lT{W#kIh!((ZRt znN|5w@F-}uPCKiO8;|d)(!g@4Ceir^05kW?*Q7FF#;ghs9S@0AA}tmrH}+Y4Q_-Ki z(2tyiRsB>bQ-9x51$sk4JdEwiMw+62lqq)KsNmo>DOISRu(@ovnfNIfCIj&xh~A}z zW-?6x1OM*g*ME(UOc7+B$J^5Xm)>jWcw&5Kk2izyJ@RZ8t&}EaUB(ZMIa4ySYI3N`@Tbd6OQ3f;)E80EIkK^!$zIa5mj#YJFkOJ7IH;N#^x{;?$hlnU*D7P&H582!ln2(CSxcKX$26%T zrILuSO!!jN>*S!7p#&=Xs5Lt~?o%s?in4@5y6%2lA# z5Zy#IV5sRTjvuv{v;?Rz{dJfp#F%J#w?#inT)~{~&>SF1JKhhWT5l_T@OY|wl0mZ` zMX7;QwoAePq~aJ2sBZUVEIM~82EBW%QJ+Ux@8A-Qg24~Z@9r)sEMRU6V5-ltfYZ) zu_r(I-+$(;PO!jqzWf}^QRZA1QC?+#-EIWkO|Q@Xzs{us`2h9wi{zxbbdOs4?~4n+ z)q;SF!MTf~9kZz#zq;SI8{baFc1ynQm;38&+^N7*lQ|p=Tv4F=#@`k z!U=A8vLe{s=kCN-$y6MGk@Yd#$olRRYR=Q4>J`F&`od7)>sS;f>PR4}N=fCb%ySn` zh&`2z9Y=7kwRWmM1nKpfpq}bc6l~1L-5FeOLgWM3{wBWyptLIM$Suzjz8#J>KXV88 z&(RI%wvgQE5uaft?9nP%4<2^TNwlhsiySEq#=6km##z_`aR!b0B z3w_+waq)u#Xv_5}KjA*~TIVe-P@MMu3GqnuOd*(8Zo46#z&sVb?>Vj$9};AB4(db8 zGg*rDZ6ff0q4AYJv!lZ-AP9Kr)EI!#z;HBVcI5M~_bG+|XW{da$v33n4LAhYRWr%i zn`wfM($-pM)5mlIn)h%j&_dwOKnlzr@a<1awDycL7RZw>osoi(yRLM%ISE8j$J>wg zM}i{!VQ5LDY< zzG9(mvw8fr56{3|sp5o--YJh-H(;b3*Urzwo`Z?=o>&duUMu}Q9V*WJ`a*~#n?J&Qj< zP#sOMKf9oWzBW9gYWzGWnrs&yrPZZr%rF;QPHj3<`__P?Q@i5GDLau{^RLZsO+GKD z6iutwg2k z|L9+4RtEO}{`?>GFAFov|E7Ocl(n_%NTdTgxq<>a|MNKd{d)`9xkB9B+~98UsM|r_ z`1ahsTyGw_@ICLsdq#CvSWIWAG_GKM>L(&ol`lYNu4e*}TwQCOrDLYK0bEdKNRF_w zj+o$ka(JM6Y8v+GGI9OkXfH#aTLU$+B{R7KIs!BUng%fVhyOxD`=w`QV#oZ!x47Lq z2WhBt0+jzA&MPS{{-m4aBLG02{eAk{K8I$O2^kssl)0 zUr+J>CY?;(p-Zh{>cQB#0AvLG*Z_N^0pa}E&jbGPlWoB_J?*^(h-2y-7+;$?wuRTa z0>tN1!d7eah4^aD-29aq>mUEI4Z7Cd)uku#G(|U6B8v-z-m_IDVF>?c{~DNIJ;>RZ z&gruC?R?o5rFWNK@{xRBenLO4UK+qOID)kS@M!&v8v}u)`Dfu+{o7zW`GG#!1Nle| zsbm_PIq1ch1I+S68<90(<^MMldf)or^BLb|$T!^zHW$0P`!Cnl?_P!9MBuSuOf$n7 zfYGT3w$k#+-7PeHF7^>$QIUwb0+EcEIc(YlLM!CZX@tp!)Rg>}aw+ z7&ExhGQHaeHWI&!Us>ajUsu9kzPPvF@V;NUqwim`8{gG4Uyh+)-`Pjs&=kAodIkCx z(4S7oyIfZoez?0?62sSC5$w$XEAC&p`+xQ?@*5uUZ!G-+?TD`?ftBgK8XPHD$uE6C z770HtKNCq@WI0LS*4l|}?!Vsm&2+$QD}I;0UXPzmNdO)uEAeYzvgeF))yB?Pim{avtPRfzmDIxWx)Qp{7P-j#-2($2$fG&FsYnZ zUGWqQ+y1;Jck%L}3!=>h0e)DYI=@e0+BcGgtsC+oCFVJ?F0;i-p|8N08H4q_)Yua{ z%6rG~K)Q+-KR`<~5wfxygZQF>ibGLM(`L7oBJDU3IB=xVgStu+lwc&b@aMgDD`fU8wU_SI zAQVr6(;&O&+lB9&9`>QB!_WGrkwv%=O0nz|D*GJqALiarLvC2b(1179(jjB!EI25y zXE#8B9vET`umw}+kjo*MYqdOY)o~NRXonijdNt`-PL1Kjs7E39o?%fpLnIWrX{uZn z>HW1Mv&-6=;*7n)P@*q_Nq?L@->b}or+(X5p61-v3E$&%QD!bDA-qTPh=Ls`3AI}r zlsfql?KZn?vy)#VZM!1OYF6TTS-_hHg7md~H2HWB2KISqzxa{hSfCK? z&^4dPa%|@qPeH?}ZfsaAlRNVAp;^3HVV(ENR)9B4rr)|kbhb$D00p9ne-gh%S6?#`qk%_nc@9b~wj^o_vHH9j^HYJc*L__BI zwU1+?Kav?o|Nbkq@9V5}PZd(&K$ocsXA_x%tJxOU-B;JJ=$Pv~if-|CHO_D%J0-?X zl{~sdQvD)l-A?<1~)iJJtiTfzJ;A0n^@v|DVh+d1$mDm=c z9t)J5=V7w0v({H+^?u3He4=G=G$ZB99tqj0uq63G)d54};u z0_C0*K%wjYaG5oOR;GT?5aiw&+R5}@uf-8+@f#I;ru+2&#yYaKHPq8agG+~p+K5DD zGq=sjs{R&jC*C@0;fnIfzlM0auFxd`XpEFm+R#B62{6rFtPNUNV5{jU%H{1?-vOUo zGR+>zRGjFT;Mnt87v0N53bE{>u?kd_Z-7h=+A}kPL5BZ)SxO`iQ^79EHC2|iU=?-` zTd4~TYj?Du!Wr}1jdzO%&!Qiv~c`pcFi7tWr`#ga6 zXOd4}!uBxbRgWDbY`;E)@drbrmfWx#6SplCaw3<#991*&sv^Cj1LGgYiM{=knDPZj zZfB454}zyS#w&0POeYV`1y|CvG$QhZv}w_#OuX29Ogr%I34|Crj%-;2dFp1rc|?m`@_rBA3ePW8AGz%;RZor z?sgg4wiT(x>425izeq{Mz`#w}MYv?<&NNWOTLWiIiev55QRAxej#7n>hx3$TO+w%Q z;L6dXC*U!GrJ1JHB>iRr8Ir`SxqtJdy*B^Am^mE2H;>(O&ESOOuI)0gmz)aCL%5Ib zl&6pY^YjPawq;qTPi?%mR3UvTlCuKB%cff-juqaE^9474_GUtj!C0Za&mGb`HdZZ^ z=)_FZc-ig^r#RF$@q8ue^JnsBK-XK>@g_~OSS=M@j31V?U-CMb^D5lzenl+N^c>-> z?MJ0i$A|HQQ*d01+@>*Q@9zl^m-uB1=s<*3t6T*KvF4+x90a2ILk#c1*38qg848{x zl3%QF0e@}azl@*t${DY*Bb1cZn9{UiPx&=QH8$XggRY1zHaHT%34JkSUD{ywhs-9( zv3xo_bP&z$JT#ezv!NmUtPHR7Hh(=OIlW?*?9{guO32 zfG4MgC<;(w$(>Vpg&JhpNtZOK_`IMt)~h_SrH-J)Xau}8-d_b%H}VYsJR0=?!|wSl)}e@sRNiwT|Q$kmk!0sJc;rms;ABjIr~kw%t~qn4W=_aJS#oCiQ(R}yI&L+`H-+jI@02D!_J7o-$QAg z6JrHX`pfd06^;i}#nd!I{Z&K|C?)9|4jkJW&F(Y52^xaXG8c4h-ik3N}8S zXryrhvBpWqR>_Z}P1yc9M+a^!BpN44NoVvhv2O*45x{Z?E&Vvf9ljhz#e=jY`x_wg7mR{;xkomzY0@D%Mp}NplDm}to<7OsAh0(5P-8XYY$Uu~-JN}zL5rc; zGWB-S`;umAirw8hM)JQ*O#m8**jc@L8>`K%(fMf69Z5cHqw@8{wB@W?GgXrf5SZet zJQV2f(Z4ZP?|3<8Hqiup2M9!dN6+y%r_>O9b4fzbN+L5Cd9$OcZFFENv~m+qA*@%S zcdAei^wV(IPsUs->ME)5T4n=v3}T-)XAMdndm=dC@+a&n|YR)9N0Y99?31qcdUWLY_~$u`pp~^XR(v3>z1#S4bJ{TQbDf zC+)Y4H(HNJQHZua(8=+?ie_%w*ou{%+D?pcKafA;>GmwXJ}FxNQhxRP10BZSqsKA( z`UMaXskb`~`QvU*oSRSOHy+7XVUe^$RWe?yoOt#I zyS9yKJ>es{3T+9#Wi9-{z@mTd)l2!~3_IAIrSW|JD0y-aM?_2(Y;!(HImx#FGoa@h z6(2Bphp4vwRJso!ATG^#5duLS&P3L<5`2f(dQ(z`ie+ILg(pB{7GCmz*B-tdJYHgz z`4};J2bvVVFO}(-Y7-r8hEl?At0!E}^!;#Uhqu z-^oCC)TlZ9+vj2C2^w%se|qL}lt>C)ak3~N7UpDGqPn-UDzP03H|dyt)1B|C8o)#X z3(?Nm3ox+#_-QiE$gQIoYHfyl1(?~+UW^nZsYbl7I@>f8dV7@g%o(@u)8|tD84b83 z4R#h_A&QII4JfZ^KigDj;BCCEVvgRd$1A5j76HuF>eOn3TUoDlCB7Zv%B!yXxFW2W&+&tDnoZ~4c!xmI$VEGPzg5~rHU@p7bX>E??M;;4$>tkjBZ9dW z!xA!squbR@8iq4v?`B$kQeSY0cgDR9)K+Uks_G3gqPMwYDJjFI(DS6V0sNHPkiW*x zt?KJd|B_lOm5RNrw|6m=r8t+N63;0~6inP3c`T^$ByQZtOM3Xnw=`Y`wgsZ=CQs*c z*tMBww0&S-Q~LV}?UD~jw|3$5=(GPN_|s+CSQFQHw`s>uT2=KH6OrqjW29CTcJmw@ zG0WZW#(%5*=O~e?{TJ$n+jKqh$L2BXqg9$@`}n0QlYL@3qj5N%W+0}a)S5`RmfkS% z3fUbqEGQgXjw9<)i0qWEa_KKKs!p8k%d>m@Y+^9D&nkwE>XcZU5^nS(rLUj}Qj8rh zvUjXeUra&o>g#vIY31^vHgG5oc}5xw7lQnRI>JkH^Qxy*2!{rB%AAFYyusZ_ep|*BaRnknGbSetz$0bfmJn5V^9{j2*t+k#4Uya z=#P()QPa4*s3}_(^S8oSruViuBdJ|rb}x+nN{z;;O)&ASX-=q2_%*y>*c&OHuKEFzis&C0hsmRzK9uj3n-+?94}|w`ie-;V|y4 zxS0Z&98>ukTmun9&yuL8VL6N(Zp*GH9v&|v5g6`gXy0?y{w7oNgs#=8oO9pjdDzbO z5P*~D>|+hICQ*22+dPrdw4A-#8I7If2l)WM!^RmUd}#M(XRi8K5ky+L%VG$tu z)Y6*HA*C7 zd*03*R4=u&Dq;Kt7Mx4HZzN=Q24GZ^(K|xa9jEVo$FyMAZ+QEpBm*n!QB=n1G@o8Z zZR78Po8yubH{jXivCQM>uvA)ZM7DJx;iuiU|D-@~fU^U&?mH*52=2V@R3sqifi#kJ zD=rn7#b+vla^7pop1MMjER_vYV+3X4Surv?94hoBoX0f%9BfR$ zF5m)sh|^$&9Y-Dikx-7s$Uv*3dzNu!0xAHf_?v{DE)QC7mq)DwZ9biiPo-unN$9f5 z;fUgbUeV$mP>x^FE#>16rsiV5twFifzqa5VUNUU}?fU7u8lTp1>-~fInH}(~W?{H4 zu`jmB7PN1fSfu(o$y%&H;T5CR6n1*&lyeC$!%`uQW2gOHH<=E3nhvimT|@bM_wh>4 z&mf{j0W;RQ`~4U$Nr34_e(*9jP8F|iaLkOm-R$S|Rns2Bk2A#Cb|w2KiSJdSjnpe~ z`FVBHO)qUVECB_%dEHn$~r5QuE1mu$AOrUD`) z&RcTaAGocsm3zkFcrv4xw0!ke^S&315T4RX_-c#;>5jGh)@OcgUO_Rb2mWZA&5B@y zf)?vp;yL=f>xHXGJ>iU3LfMsN5u3-gXDmyLs#m~1PROz}7@9)^zn05o)j9J3l#zc- zBg}GY;hEuC98|VTIrT$l$d{QJY)LC<#g$Zc#}kS0qkJKR%~ikqOi$4|AA|r1Q;lG2 z8`h_Ffmib~Ihy?5p7XAuug{mEbr(^Z*B>9}@5HiE4@4kSaBa);TbxYTBGoGEUTT=> z_so-{GDs&A&Din1rDUtY4n-z7sC&hzO_!>Fklfv*le=0!Hf=@bBixDqK85G0clCQH zS0GfOT3_Q6h(`k_NdegU(bJ(@(pIrhJG({}lqjefSv-O=?wUNfgp& zl218hn-Kszy4%~>AoKg12^&n%i2a>_-VNYSBSADMwF3=n@rA05G5BpJO9k(i!sI1? z)a{xVcUl{rz*g(?$b4bIO05GJfj)5ej~I#!Wo>O!@!Aj8RtEt*#?!mJfG;UGI&^nq z?RD6C{B{Xc?DAGL1>}zR4OqK2WlyZ**5b7Qh|Lp3#^B{ueL!Q}R4GOc|_e0Ex*bjt!e~FKo zNQ>>^LFy$lxf0?|IdDR3sK)N*(|0CoGDOFXiu6Dod3L5QNbP)b=B7U483QUVsi}z(Ahs@?E^&!<_~&q=NicH_FQuvA&x zQBXGJb8s{g0P@0Px=IGZAT~$(%G~CS!%&EOtyzXPyPF~8$5K0=s<8wAamQENb+DEF zd{_=>qEc4PLI+&p`C11QvIpoj)oQZ0md$jSuJb-8r}l+kdS}rqNJdHES&Yvcjv7MI z4uOK#c2IHbyq7 zkGye;8msw|-z8l=<*a1CL3u(h6M#qqhN7dKkT5c!Uh^AfVQ6`#X`%+cRxttRMaUUM zQ;h6HG}mLB=3YLZNX#ul;F6oJHS^UhVZsB5J|n%izZIYLkKr<%y?f}emnxea`Ej$P z4h&3VeIrcOLqrT$X-#YSqo>GdD%FFh0?=;X@&OwhZ+$s(Jh`#dGYzeYXd4t+z(i!* z$=gbPz@@i}pqE5w$=GgqOA1?2Yv4Qmr)C;`%&_v2nT~g01R)-cl#0~nv$<}~>bJZe zSC}sE=a+K;pI)e)oQE66uFrhtpq;&hZ#8v>UmvJ0IHzpxpxbN2@9H#gwzU45PT{3$ zIqhx2PgfsI7KeSi6`;MA>^zB{*Vx(xzPpcv7jZ7KG2VQf)Q?L@*3RKRbU5+ z$GO}_Jjh@wvj5eL+tGAE@HTdGaw8>96!2bJrm0S0VrbR;eWeQ6kE1)UoC)_5cN`t+ zv@n!y9LEa90im`oHL8L)j#;Wlwh=g-jw{^=k}(2FRFbQo0C3vQ6y!bCj&IAGjEAtN z9KDg)OEk4?hzposqE~k%sEzPNb*F5OwLEHJ@+SH^3`MA6=pS=;!{X1~-wI{YJ=e2$ zO;#r5$#U!>mX*r(dbx}bD$skE5HmnwdWGIQm8YG)N@-~Q{)Q}i$v(Cja(4gwl6Im9 zUUQ$DDpBCXQZ@S6Ri3WDK<3U8`pT@kZ=3BW(E;MqdRK`s?juW28W}|(99{8govD9j zYem`hm+G?s`XxAiL?DU}F7PpY6E#y5scQSyTPsEBAVi_&eWJM(?R(yUuSG8~fr#Ma zJ0A$7=n!<$Do0K16@(4tFweWJ-97UhtIeoCAg~xWE*K+Ws4UYthKgcur|`KKg<94f zmRN8ksFTv#v*qS9H8f<0Mz#Ev)v-p~O-Qujz|AXtAu}@ZkJ|y33>q&YlF!69Eb`Y6 zHX$U^a>X4@n$`KXY@1|d5pWQBIvqeA^!;ail^e?dA@MlMX)bq^J&KNiCw_q|@n#JQ z|CPW3guMIM+@2P_;2lIzIIdfFi|hJ@29wv9#Z>|gEGuf7gVVIz$Q^nTMmGLtixAUc7jal zs>v3Ksgr^hfm(A-vX20OBiYsQCP1Wq(IRul5-muE`UE2*tiq?Hh1Pkp?xl5kyVyaNhvocE>EW!=wD}pVj~Zv6wL~q2|rr5#b*YualYA@y|uTo%}{(MZ{@k^liTa0`k z@5!fw^F+QA--JJaD(}JS7?9J`PFKS&Ks~_k=i{gfrVdD?XR4=dcD+A=*j-w0DF0%i zER0}Cf+5@pzhM;d+}~*|m^0Zl=DE(nbQmZW04-uaf^fth$k?{t5kv`-~1g__qtgL=b-H>W8izch3oW$AxIOnMdmVy*TL z>iv%D^3Go;&7I;i;jE=PVKd739ZydX*gvc6M*?+3=s8G5kCHEJlR;=j~&Qng{jPC=QX0}3OD zhA(zqmpj|H5vRpxpL!=SVuXY>(8JK)Hd}VOqfx$OlJoAX~|5|tek`kX%T#I z1{1>POp?N<=NGk=EZa6ox+DVqoWPN`KE@<)q5eL6v$4(LsMCnmHd!xnIT(E_dL1lJ z&rT-F)-h^ql|n~FniivaR0U*>L$arX3OW>LdF?u11eA*$z{&8Ygn03tv^p)I zo=23vJmDTbX#QL&xSJFVou)73bJs$&Q;&2H^@f+}shqScT?0NHPB$`j>{lc+}8`jK~ zXK|Oy>nn?yl507Yszy!O$Cr1PubOy}5OfLaq_N$&U<0#-V+w%{-#xUoeAUYiiMuq$ zLBWM>l=lieLhLi?EQL>?-gmI<4b_*OjK4pSpL<3@zH)vie30VRoX$XLx^Efb>+zYv zW>wmK1JML0(g`}hV7GptpkU;1k81;NH6Q6cU;Zz_~U?xN|q zXXjudJkFC)&2Yr9L~%<4On$?BXw#v$Nj@>UaE>-_Z25+)8W^DL#e0>%$h9{_l<9l# z8g^&k`u?Air?sC1$d|qs&69XtK@z|vlo>pfspZ%OR#U0-!8(S9W3Hb!(hs8NMt-32 z5y;Oq152y3@TwRoj7?=WpH(=gswjY22Ry?iSn&eHKEvfC=RQ$M8Nfc1_<&7>FhWM0 z9SC-TR&LWAi&b3$$oDDm!}%q*lk}yWOA{h~*JqR-1Xlriil|}C4kViK$cHHJChK=Y zc^o}Cyd$kr=8J2_X~S#CsPdB#cr8>V?DqdEjHGy&w}DG*&pqp+)7yLV8Q))_kNBpz zkKHsR;f-4PxXbc8?j>I87b??9=Lg{%q*Y+mm16v(w>9{#7LFFy6?D1V$&9Yau3?k` zpDx0r$hM~a$SF2+IuL}cgPjlJdu-A~MKWp1b#eg&n}4r8qtR?G!b24=D_78?fl3Y6 zO0mK_cP^y$1r~&U%fJhUulw6R&44>GbnPQ z5h2`oGB{M_Ro3A12|4`R5>*dHied_-{X`=^HJ1uI^S29xd&Y1BN=L;LB@#HQKVMdU z6pSy`_~&w01~ccFmp3PR-ouW8giseD%x-Xpd=R}$zog*V&`od$^6J$M=YiXY4s5KE zANd9KT`wCfQxd0Y82HW}-t|P6Xn0Td3IC$F}~ zaWED2fQIEb{v-qQe5Bt7_8SMA<|plN+N@@u%#&Gsw3tT{+($7YEV`@ z)oAAhtmL<@$m~Y+#E^146{5=gyLN}-3+%5#cVAFg2<=}m?>@1|MUr^w&&*)08i#^1wV6v@K4TW2Tq7>kPmxx1kHccQX zYhv!Z=Jfemx_i97i0e0t*@A!%g7F?G(^C{R$c@oH63IW-VXR0M{@eXbkV>7fglgWNW`Nzu0eWkjH|%*yx&h z!zGY!hbRZJ3(%v|%DwS{AXOeYpzhE@RNW=sJ9twnzNxB2o+v!Mc$Ud^yWc+u5)-*3sgNeov|Cb$N2`?X$(R+v@JB@A*l#vT{pCj5j&wd@NQP}ZI!1-7@7T-| zO5Pypu8o}6e1oxj?v?LHpM4UCibyeE-~vKSQ{WjHrX z#wFaV8Q?%zd`2VW2fiiCmr|7`j?CJWBwJc>EMps9%Iq^)`m;a>C1lfEHrzo~@Fmf8 zqrZ?@tP{Nd!u=E9bm@$NjEoM(PN8tCSB)R%$ynwx$o0^8gx7lz!#kM=(P)vPtivN5 zlQv&qW!nu0rGCHlw|()I!u--l-3j*2j$W6>K=9;9%VsAU5;%bp5}xYn3^4`dkq>Fx zXu6F0V*pFzA!`U|;G{x!;L(Lhs>@veF@Jr+5D503DK7StZ^!Plx znR3R~Qg8|+VUQcjwvIR|a(Qwj+5q)jFUdAg|W%M8{N9mXl;_F!mnKOox62NE;w}yk3IqM)1rdD(qbT0H1 zvCn~oL!|Geabb;z8g{f$l~|OZu!@@jds4DUmm!%z@3=?sAr>VstJfS)Mz%rwe{hq%!^sG)@vr!rn+7qkR-d7 zTA)EF8GM6ph!dH;M?X%&*|cOR^6i!Oi2@`Q#;^9(5zg)M&JZ}pqXIR+dOqSIhkwyR z3bgoA&Ln9a9?cHvCW|;jTHgQ7fqfRjaC9j@P?_V<{Z)L=c(ycQoOp{|RCs_;_wy=* z-@H?`QYX@d@Y`Q>&dE+SPI;;?;1*FN1l#!j{Y z@RLZ553FX1hb3aeZDjI4Y82qM>B0ODf2*n42b7CF2+JL=x$AFf`EEuBGJtAGi3*{0 zpBOreBxT)DE27is7uACHJQ}^}qKkuLY;PcHf8t1?d3P^W=|Cg&p9xZg^{vT=P}8A9 z7fuv%&uhS|z^$HQVKZvP^ zN-;#|kL>zV}BZEo5jH|fPSJ+y* zdQo27vOed!X04xSlt%^;RzX*fvVFVS@#nIZ*yG@to7p`Z$fXsDyHJg|@4w7nV3S<; zhn4)3lB-uUG_4^oy6^Jc-4Ko?lH1I zgdsVqo28dc5fm>-1R3bEp|~x1bk>kuzmIeC7kDTCk(wm_!fnSG6Dfjxt2a+SfsRqL zu&-MtgLHa@MCfvIgR?0L49ll{`#|@fKqmt0s%Cd6R3S?NT~l&j~Aez7f0qaJ7}L*UW|Z;}xMu-&&+6Bne7 zoY9X&X9Gw*41ONup_hmJRd3t0l-YstQx<{-!%{DVtH-QxkK`2 z5F;$EfE1m>p3UPJ9El}}&fFXb0u_f@ia6V3Vhnev69gTnZA;~@KpO{&$CMj=psJBu zN-e)t;*EMV=W?-nJ_3vh`x#Ad#h}B0mQ}YT zGzG6|^;(F^qI|M|nZ}TaZYN62=cdvyjbz^GHcDk2uOwW8H>yXcw7YMdc8ss9I^Hil z>I}k~_3m#y7+$ATS$bIA8?C%Ds)yeKDs?OYHzSI_R)S+lBe4-qsqSUXc(Qt+Z8#~qF=2ux=Roe{W zz>f4%g}xHhCx9$1=y80ZHJRnibkJu000+>n<`nvase|0LrM2SYls#p%0=Yyy4JCMz zX^07T{FOa_{4q^QDV|=Z|7}zrVOiCrHx)g=iM(eKjNIgd1MWv;<=tsn#wteL=qek&nxrpi`b;@QD=#s{{CJ)IfGG2|dyO}lmvZfRw-!cg;P zf7c+b9bfl)A`>ROiK#Q2DoDLtIbg*%c8NZ(l(H>}?XY-n`_GLn(9E%A z%&7kUUY2;vA#JmpErC$fyp}~ki;YYYrdbVPg&q$V(IGq z(5N5|2N$R_M-$eb0B}XJHxOz>=v<2~%1XJ`X?O`KsuJ-D!yh_SmMi(*;%$pazlJiM z4VK+)oM@8T%h}<2gqbIK4m)Ss{b0ywu$KzgGE_v#1X%~TE$bSi@d1y=dL9VBkh2OV ztuxkV(t;VUe_o~(D(OFMKBEpK zip723N_zj{R#=mhQ>Usa^dm9~>o z;z)KW=pu%!dgU_)FI)qv1>HFx;DR{Q5SBj9<>#p1X=6)$)2BXyT~L5Y+=nM`0B`0{`M-@?A$k8bI*0&t++pANUO- zGz>2i@ds7mB4& zKnOjw(pgCLfM>fvs|Tp*dIobVBVdwdd(b1cYZ;^fZs&!$8joA(Mx|QDS>R|RMC)Mq zv9aa6C1d_tScLfdnd*Visg_E78dr1L{9F8R;(3cHe8^2+q|jx2GEpa_^85{~iI{Fj zAtKy{esPzHv`fn6z^uA);o84|T!e+z`_kZ=Ul!|HgIqmdT(w0=v*aWOTar?qoyF9T zT(E=kr(%w+5w1a%2%z=-N&oozyfsJm*2+oVUsy`Igb0+?UrM)-91(v9ri8$kc9@(d9`H#VIk-&_MAG1m5mC92=i?sWDC9{JVFSQ_-Xx% zN)O)1?;0)#p#t)DKhkVBZZl#Y>Q(rgft`atE^)CT(}zxEb%Lb9jQZZDKbZx>cnh4s zjpsWG9sEhV;OUXY${4g@W@X^87* zh7jTr?pYe-IYX^bcx`3mFK1;M!X~)<`q9%+UDnhzi`17q-IIzqfPOws1 zqpUa=RMuIg0-xJH;f7s=N~UdK0|lE_%|}c>O)rqNoFj~R)Y-}I(yY0IE?1zqg339( zduUA4mNLB?pPa<$rv$LOVJ+88yP8@bG{Ojkgc#1 zH3I?tD@B3pHljy*Ns0Oy6jKqV+=sl{JC<1q-%F7fum(B3CCP#;*bHT zNQJUkOE|?L*-a5q>`VbAVm63jCD(a`%7aZQ$?Ca!RI(%?6fXeV1nk5*v{0FW zp)+XR1fGEe(sp_WCGePA^F!1K5^^h7T_kSg?NW|4Xz}Lxhn^UC>qxc_x%LW`$h&`= znRYiaN}Z<-DnAJ(?ZnG)2~@9RVeWl)Zkmmu1|#zb)mH)_HCioUFL@j2E5%^o3Y4U! z1YVOnYMl4u2#j$mCoiisA#EFE%=Y%Xu7vWPK0d*v0Z)83O$9}UokrzS zae{CyXOyzW8Jn!ss%9Pc9Vs-Ot9xW0A_AL`LYde+H3Xlg(_ZI%uUUPMO?U@)}P-!bn7 z0s@9!{O2BT?34A<$($FgDG!HDYL!r5)DS5h zp_#+>sARH!?pf)>1GFsKvh*u#k?32?ld`B&qEJL+%Bn9uRh(0+6{S)P_uWUbz&IuF z7mVf?3TDKI2xivY)FxB%G8?AaT#l6hFQh6@H$)A=}A~^5M|ND>jTCyZUFdhDOD+ zaH8<&M@qT2*ED0hDR99U4hZ2h57brCrd!;y^4pi&fR+_rB-|n7+JB{_z;s;O} z{T}!;OsE#|2WwSz;gGr-TrT%JCqo1UzuemFkOaMpPsefE#g}UXo>@eV;VUY!cU;zW zo3UUPK1T7o+Y{DA52G8ioh~#BIm%8o_`TJOZX`}#(JhYCU?;pP+G}_z`CEP~-y_dF z1*2FR#$0-N=>rmh>Gp{gH>jc;YyEae9%_@4=oFr>l{Y)gbM|osDdR_bSvy#W!(oj#?XnQP`$tE{vao^ECqAZDC1 zME#H$Zbt}vO6F$pnQ=1*wh>2e@$RsxKsw_6bXM1v4M;kgzbF18DH~)G;+Axh-z2s} z=wf!)U_KV3(HwGMHAIWg)-2IC1kE-V11W8faG?XhzFbLgfGsF38#ou()E4pHE&bxZAflG9NuLuN!>l=m0&Ms6=#20sYSd2N~KwS!*7lJI=MX^Htc^g z5Jy$`&y%DlTQ}%~)`MuyY?_7jgh*s~5$wp2T~RBA84*~FNL`HFZJgdBzSe=%f7>2O zJIQeD;X0Km{WM!p@xVnjg59;`aT9S-4yB<)8_z;MlHdznC2)$qL;6zd zx9KEuE+_DS^q+rVq*tA>6DXE}hiY<3`n3mjL{m^dOp1FIiIQ2)WGHgcZ3*&}a^6#a z&Htm7i&lm*>Mey1rG@fsx@0Ic+pr} z|0~0-3tK(-iFSqb5HaZ~S*q}K9W?vYIrABPM3n}oAph?b#;%_v`rmcEWm7S@Nv2|f zJjpb4Gko=wRsyum?%wyT8wXoW`oyyu*;xglnT-E_<81Sq)y$uwvK9FOeB@TSrx z7hUj0V$|UY@i@2L<&EZ)=oAIN8_~}a0oQThsmtk(BKA$Xo~JQ)q;J~Ya|!CnI3Ww| zpOr?z1j1QApe@5QG{*BV7v5oeN==~^v^Nd2smehk=ot5hZH+Z)ruFkC2pP(h;OoYl zm3C(YXFcQdm3-K7cZNG7!_ngpG{%mUGEuh) zp#%a>p)+0XyR9x_aXT4sy|O+KnvxeYaO#8LVH{?PM(7`)ZEZnM?Fun#Q{a)3z=j+v zOJo>yf?Y8Pif8GCKwD`xW$iYmn2mNHxn|`}(hJy|pbMh_yEM?M;gB?O*wa8WtbvDV zbs1R0-(gvY&qKU7kLRNt^JVp7;f3Y_v76+p^_VD7p{EB-lVH1m|KXxmKEaURx;rm8 zZe(c*r8O2G1G~8Eg?EXwMa%}<>5u78Y{KlVU&LHZh*p>A{$HjFV{4EJ4f{1JJ1h(` zNr!oFCYHNp7W##?r68!a2~7#0mT;TsW_|OtiR$k?~P?{h&* zK4KHI75w2J0!@6VV@a*l4R1D&W7W&6NuKQrZ%XZ?@KB8K_CsHbBMYJ$^UR*+{}shC36ww0hxw9iA@6!IG_CnWR8}Z%3`A*@#LR zV@XQN#7x8S9)8-F=h(1}-h;&T*s`>UoXb_!aKXF|LY+|RqRiEpqZB*M*i@r00`X&U z;Ae#dm!z>}81W=U-94L%&DGYUINorBUinud`9_>F1_`|-oqy6;^xZa7gGK`Stg0G^ zy{ry1Jv%84>5y{{ak=p47`)Oq7B?8`l65W-8ysVL%k^)>#H#0=Vje2BbG7lOEFO-+!B%3DAt} zNf_7u6fcE=HCAKc(488Ztw#M^J8$N0>!_VO#@u-E%Mn!OJF)4a8iIyN0^Q!-g{aN* zKVH-(^G&LF1{=?>dogs{>x3Gsz}v=rXt7m$l3N&?|H$A8ZQOHa{`1o8*_r$(04g4|Yo#{W`Mat&b zdVpB<7?2UKPf0JI*I;SOBX@_lZbK}!SAOUX2!WlU-~s-fLYQ`NJG%IzENc73{+JM5 zf`?bLc|7tc>2XA+Ft=v+gi3e1?1x1zLD2D%Z05?A9p3n;cy45+?t12Pj*j?z^)k_9e%D}ClU~Wn@am<wfyXhV3yii5r|Vfx}vO~YjKME*sy#tIs}|1F72hF1O zZL-0|6@h+Z1Yz7W6hWezx#IT(sw$QaQ2m*Cm6C2;ts9X#S2co7U_P zK-y5Un3#tmN&epRIVz$Naf_ITG zCZ}JIl)l^NHeUvsVSd`%-ro)K%!IWyedw3ap2tLd`U0E!7o{GoLRGV8GUo!>bR+?c z$C^Zt2+z64%m3WDEmw%r$z-3k(cOx$gJ;lleDI!xdRsf!IO|U!qK{1w#Qm#i657euo*f z6VGtD{&53GL$!Y&wL@e*n&4(Bk`(m=QyH4#Ec-yIpWtY(!;%@~p_nyPv1f!OLM*<* z0T3%-WHTi_F=gCzntXSt6ahESao4}QdExphx4K{m9QL`8L)L@9VBATG&Q;Hk4)G}4 zO^UBq)z#%Va{Z*+4&?^6oq1ie)}(G_13fwCul4YfyMzjHZilO^Dkfy!x!r1S3_LW) z&4tjgwMt+~OjAwK0}sRVJq>Qb_;)vVII%-84aTRd5o$Aqqn)eP@Nw>BIpU~!IZ!Mz z)CmVwvggg0)9*$0r91b$fM>;7;JdYZL-z@KzcT_>%YGN%+ATfy_t1T}6}_WTUg< z$+pZHfH}_`iwG`(?g;38qjYP+J&+eGzO5!QIQw4SSB4h*6A^%L1xzG5+O=Q4X}7ud zfex0`p5U)_gP4QaW%#W!kLytNA|__gdt+At<7 z!YflF7EK(Temptjx`O%vzW_-&>W^inqpI76nl_zjo-hQ&w%eavFX+$U)utEH*9y5} zT4*9lfAoWu%=DYg#(RepI;E#uf#8!BByIO5?^xPI2mF4xKRP0u{AbNXko2>Lpn>paS6pUGr0 ziX%FUG*I$`3OWRvf9KwuUlSB@UEEOf@e4q8nmzSTR*3fpQxYHU+;Em%0UsVD=YlS# ztIx#F(FZ?43mGEcfela}-G`w2*0G(vg-!p0v$qrP2X{3DJylE>USRx50j$m~z- z%u5@AL~y}axx>w7=Di$=0u^;?!EA-Q?ejncMO5@X5zW3)Zom(*{#t+aKK>LG)ayMo}*(Qv7GYX^-NvuAw z9zLUK+lf7r-B=^G+y@U-v!#Xx+2f{5#RkZ)M>(S6?=;;h?rt74Eb5Ps8`;WG1&bV8IhcEMtOhp*Kj}$wYM2;~X;X?z zIqV}!tv^EFy;7~x1-1G)^|UAxuY4{j-&q9x@E8Grbbl~AUGQMq@yJfXadh6cs*&Q< zIFS@vO2PG=S~*xXBVp;kltEWm#YY1*C@XTtvsf^b0eQY0YGr1lY{5rT=sb>{H@Kt8 zV8{p+BR0j%2Qq*6^e++;U=AGGQ-Pj*U`5Mj_kL56$*06N>MPkE?{D2T|I%(hl4`s| zS?j47lUaI(Z-ZG`|Afp+j}=2|4{5o9^w56riQ)|CWQ)BXzc!wBJ#L8Y9&(D`&LZ$h z^l7eY9pf#aWa*WYD3$J18`11#?)K6KrvPZTTVTXT&g(}&=kMH>4U!o>@%EWjB`N=b zuT`s?ot`$N^|y{tCx-Kyn7diF&zkN_i`3=bc$TH$4EdXMm37A}p#X#f#;XuW3Mx1+ zrv2f&m}ia(d+;B@6NI&LLoQO%8PmKLX*tRM!DUAE1j7+PcxRA>7KA_Fkhb4-Oi4}6~DwZ zmOX6TxZn9lnn{RHJu7b_R)#glgaogxeOTo0-(G%c{kOPhK*sgfLlDQZ)YnTLQ8@^4 zFq0Q<`q<~`@`Aj8xc1fKC0>W0)Wj2}Rf2?rWH0xLZt#~)WRnSAPfc%UGL5jWjfDTT zt7B!%sTvnH)LBpq_*KS{KMq45KnMa7@`U?~N0-#z8eZ zw~~NDqRbTAit^h?jcHdAB`yV_N*Hc9o6ma>0u!&!-aX=WssFD`FA8HHK&f#SZ_iK&ZQG!RQ^;rNV*d4OssJ3=+oukye~>u@^@i&9wvCDk8l59*%^c zb=R7fFPt60qh2dBn~uFCY}xD)&U!rfoFDqY%W3>^dOIS56H}zF5pw~7`!gQ_Ye!j> z7*U^G*o}6G2H)+EFzr(k)r5&n68pM4-txkqHTRgej{T2jz|sj&ybpeM)dnvBtDNOV zDj36l=UeH>9gx_@V9D~%ok*FEa)dP$!(>XZjzBgZV1OFeMbJm>eDQ4Sr+G zODXMZ!V8=c_M7IT36M2`cxI3MdM+~+@>pK?1mO|4@a81ljXhN$`r8uMc5@!XDb@xw zyCpaw0?9~Aq-o>pdpkiD3?Xf9qv{_XQoxHoAsFQ;)u5TdbDC1Lk4APCc~UwOYDFeR zyq^QN9xOnm#z9pNGD7WcJYV1zndqshizCvMz0bBw-DsO1>ifrK0N_ zxWaGgl(|jba2BxYN!$=^FtIdkiL8Y6`n2`6<3vM`K&#kxki|h>#0WB@duUV^3}IYm zzF_L9<3uZ|G2g8`u~lqPMCEa;o}RD9R5o1jNj$xDYWdI_pfliS6jvo@iBKa^oTx35 zVV-3+VE5GtaBP0Eu6p$JZUz&`VP9m|tK0SzR32^P z{WKY6bum#Rjyzg~6k49h6JJc9Lb7dDa2yZg7X7C^e8mw=z>Zi7VEuWd>(DrhS@i8Y?iMpMTrnC1_ z^!95I^@Un^EATr56gz5(K7yq6U%*~k43gV=>LV*qv1iCmw%+W=%CLKX@L<+|UAphB zo;P=e__?j%$=0vfod~U-#QAd_0#&Mm9&%ti;#Jt-$Xkxi>{KVp!ZT+)$bDTCkdd(W zxq0F|g|~XOskNmsd3a^;wF4x`#p3jz(r(paXHe&y1}bL{(_3=DpR?#bDGLo$pvpz2KZ=QWYljdO3& zgW|VKNFqrvfVbmMQ4LVT;$zVSl&P9!(l;Bz6$yF{Zz0fg9`oj(bGWAZ%zpKbOG3I> z)`l!Z!4TYn)Z1TX#*Po=2JokO;4+Ly=ptclX*PN@PWH}EN$raN1f1%Y2(2x3Y)Rx{ zstJ=?rAWo|(*AdMG~H(XW+&11PTz;p-;qxhiya$87~y-x*EDPE7GgG+K&BFcFuldC zs3tVq>C^;(Ek>GAlJzUJV{VC3Ih<24^dxw+=atw@LVC)^f;F3ZFxA(#l;#3ou9xD| z_T^3>xzqP5n+^vuH+e^SRAwAxm`8?k#NM4nEW3+`5fxicl4qy|+9WkX@H}w`U?P%J z9r>)wgWQraX^FC#z9-54T!k2)`LyQTlk^x3 zfkmBW46EDyp*?P&o1u%8ISKss|EXikiBW|#q0lkRdFy$myb|?$3FaB?i3^3%!RUwI z6RQ^733)=xSQU;xLikMCA`nBb5m%fL`uG;}TTEH2Jhi8wdCm=oMeuA6ySljAB2Zt1 zK@Iu4E{<+G+rrY!f{+5VOh{n3n;+8pq|_L!qqq_MbWWZzvMDp+A~e4|?z^4pIu+8iQRasjPy+C(0tE~b_D7c6)T^we4;Lnz+$!Q2Ct+#WQv1iHqYZ+FcoH zrA+U-dP}5XYBw9~Shg-=w#Ri6v(NRj92iw2I$ZF)?65aRn{JmRK(Pje@ob^uQTPz- zT1qc}B$EvXW|xr-X4hLJ5Tuby-Mqne--AC=P1PM*Mpg)54=h=|Y4pNhdxRB<^9aaa ztE5;O=cXFdbLf}@;DBV_0Cg1tF3Ns3^gfgr0#A z{>POD5zth6Nj|lvPnS9bi4v^@vC`p}$0fBSlS($}{MwOGjUAloS>8kHm^JNaZRJIo zKC|2#qH&O4Bryb&W9ovGNE-bg;BoiwMbI&2$upxk=^j*~B;yXR#lH;ht+#^zsVPdZ zbH8Y_Z-o%zc{}!b>?@urLl_sTUCW0^$6G3JeRx?FLnVt>C}6NnCwp_4UKcR42Kqg| z!BJavJ+pOqA$_&&-)7V+oZl1Me(hof(`sKnMmu&Eg^ffGXG$`qu2qgJw%NV~NLdmnvwAb& zM~R5@kXQ)Af@&s=%1VY%Sx>KSANGEPe42EhRw97Ed%AJS0G+6Q|CN=%=PN6DE&BNt z`<+IJ7#Y;#LF)m4zenkzo4|9Xm;u}Vs{7p_IHHV`W28;&Y0v#3$1S)li(87y7anP{0{L0!IEp;GLMxDbi+WDpBA7X zgG5hwj%~-{9hh%BZQ{|&5PGN51VVd&Y|J{U*Ky3QdhvDGNFSktD`0w+-g~_g9ewNRDX)1) z>Z-L{MJLux$yKu9=~mUD>NIf#MDlHDi-F5JB;=es5xLiebpm#t%8!Je8s8N}ZG`p! z^MY#)7zr57HqWpE3lY&=Cs~2Z+Pdj0IMq+vsY5zH@ghGx^L6@o^fxkunRo)Hy_?3! z-!N>Y?0zm0?VJ1eo*U%GJjkCogQA~6+XD=P`AQw4V^iQhw9%p}Y0o(bgEfH-1|;e` z%`Lr$CmUVFVO=P$!&9VbGvY4y@HNA0U)nI0mhcQdT7L`P=u&BCzyLtW>P#*l&ChpF z0M;?ZZ(FWi5D4$`Z(~x@j*YAVqCLlfm+S2JO=ZB${byrts0O~!bu26IQ<&1`4^N+kff=x zzlbk?EpG>O%mP)-_1V?CUq!}|{r zex>&QkAd+co0^(?0kTgsn=;E^cpkX6q6fu}ra<4DqS0e=jWgj`Pw69EJo3RxE1hHfwaJsr9ZpmX#(x?eFJ5|;kla|(wN{i#xW z-Ai;aM!e@|^4pN}HOqt*X7d z4k!WUCN)ga)I&YS!FA=bZW3o#H4H54laHt4PU4 zSoj8VH;Pk(uW0%p(0cD}#KUrj2t>S=lI&#$IdKKzn04&ib^2UZr4s1ULJKDP)I?lR zz36Ev`2v!uaX?7d+lBO?#y3+PGd><@%v3cidO zX^QZnTAmmyYaDF#Z`8$ZbbnTzaAXteU6xb_{_tPf?Bymvi{h_wX)Dw$IZe~I_<@D+ zC%b@PMfU8bSrWpKN$YG6VRYpev~Onj8+>~*GvgIanG3dJjs>cP?Ozc zc4Yx$d!t?od*m8jlIo|ie%Q_?I^}y!kQv}OY7@-i5Iosph(O#cD2~>}oFEa+U9n|y zYbvNs^~4r*?4Xy#&6|yrxI^fi_Ow$E_IS~$un&Fua=m1rYPE<3u)Yst%pQRdMn zV1UqITG$;8JV9xJlI^aj2w~|wB828FxHvo{p}c$SF|J?bP_mw(U^_;kxq76LBXb47 zf=jWR@L1*j!~WCETYR&j_ppuS$cg9tHHcIlI+9cnQY}bG-TF$2_|eTG_DQL`V|%*Y zw5`KA&Vc-QWgi+7)D|yn+&k4ePyQt+N@>n~oNz6A34Pf|K{TKHs-Tnzi#ZU=vj>C? zeiha0NNm_W`+Wj;Bi&(S4@O9YC^I&DBxBGRrFn-W=CUFup5P+8{Bh78G=A0s{YVue z*dgtKk~?-88(jD(tgv$rMnuI_oxOAlPDf#GY?U|gZre`tEsTKAdM18ciRAT&q;;65 zl=G`t#4o!`Bpv3J!ftK>a!S1?3Qr8Hbe7URzDAxq6rT=70~2cpo9ebUOB9mpv7jM)F(!SbZIL5-_j`b-MWj*TcqB_n+X+N~)8VnnFowr`l*#hurTpn$ zhw8jCS5f;Ra462?FR#aLVQ+tXqx2tWq9j5pO{IE`CWb~EKPdVmXA6Du`ucZ*K5+Mo z!8frj&0XPiUVsbApJCUUUS(#L(F~j`7e4@d5abip%r9}F99eUFup#!bSxHiUPFvL^ zk7^Sz$?ex+{XEzI3ptSEv!^YMOp+=8(n)bPPs(B@Ky+mdIKRn*kO!pT`7*QyripD+ zpo__hROxXZEZ$Yu1`sYt67{@3U8r8Z!;+O+N;Ge;BSv%8i?y zdueC%-e$ZQoRC`B7R*FV-skkyDW9KTOq(rgb4G~X%r>WRC-X@8WtM-^p`cI?KaJ$> zg4*H5(t0%a#**PHtst>%ldIO-qG&9fQ^%w&p}|`7_{;CfS&CTt7<7T5#rNHoT6PgM zTfF8olEF9 z+ab`DSEhP;)K{@@e5R`BetnCi=5Dd!&5LAwV2JQk5%JS6jXA8xH!t^T3<+L7W8~vU zJ^ma*%l{^Ko|=E5IA0FKIHD!D^gGWj3n@Ov896SZ2n!*FOyxM5YDZCPQ#G2RZ=s*7 z(hq4n5{W!Nd~6e4;IUWyo=E!xnBe?de2(xKG|dPNVhPw!(CULYydV9D6ECmZvMnz) zRby6&Uhu}6rRO0w&-sZ#-_-)&br%wiv*;jnppY!cKfUPR7EQ+F4cx&H@DJHxDZ550 zrfbzN3tJ~Z%&iWF@hR2NxgL-(U6Mqy~ezxxjyC z|NO`Yn3Ic(i&{ts%@O8cVqlHtI_*_$qMBm)9TJ9&xjSFiplye`OPTV3%zF0WE>gnc z?k-l_9AX=NT?78&kG?)GGt>F%<}sgoXSJV}2_?xPa$4asqkZ#8OeVJLeVyHXU$O0+ zT@LLK?KmoFm6ZzoQmlP%-{KT-zll(w_7J%7;p=+zrh8zo#=rc?BP^+$OJuYQv31~~ zzR{2Mm8JDX#Ini?#~ZzjCOTn}d&aLVFskn5Ymo`OeG{a*e4>vn74RtPU;&CEE2(L>K0zFmQcD`WU7 z9}OkG+r5^T5#X}-(ALgKt;DA#XKBSS-ooIx_tJLv03twDaL_WLX39kTe=y7V^4#bsn)uLRb>u?e8-^w^&Yq*#%%!s3$+0sgQ zm_;$;h+MWjx?136CMHkz*~q&NT)geEwnDEsIJKgFjKmb)SFQcgOeZi9$xp_`}xD?yncZ zLQb+#1=Wv+dLOnPzaSnO*zrMtcv$?7`h{F@9kgR z=in+o$KFXD=^UOUIMYk9DyF!7du#vH=SSny+s=|V1GlNI-!5%?ZgU!i_0b%-tM4UE z-G(>wB(7ffwv?{#RZJnU68Svh96D&(Ui1D~26n!&e3<&P&8MfuakBBnPK)1^+imc4 z{fC>m&{JvcsMOGr1*JXT&~Pr)E1}sIOV>St$w^%EQ{i0uTlAas_W9w{lgZ@7X5s_( zsU2ZX-UTn{?ZND2K}F*rzSZ8@#Le}`=~HROp-bJX^_05#>{P3;)SH6amiVF3t^?hm zBjM6hFY((!C`wcMJ)8|JZV8^#%jc)2a0shH@d3?n198*)?kT?=(X%EA6b2f z%pP3IPp98;pulaBLu~D&{r*JVIPq84c3t&7_9`~f-0P{!NQ0EEJ{y7MW$uAR)wXi; zYXmSNZcP1`w9`<(+PoGBjK#Q?RV~5~9Teg~x#p+cQsr?i0vwV4@<}(Y>9q}Cj<89k z+%TDv-c{&&zJomH`oR~2)Wo2rO)&ec(z?x_x1sTirs%KTVCsvMXO0B!n>H0b9x10f zMT%xFTWudZC%WhYn!8BSGf(gYq~p7E_n-9E`qzSBM`Sd0>!CRxZ9l28; z@yGn`-7Jnb?oO_ARZAwe!R_Vst6mwrAzO9|MCS*w8;zuw8hFLlua=J-KOqvxLq?zm zDXny}4hE~kD5BZCr_rqbv9`RAXvkS+x0Fk*c4mXxEBAVTp#iRXCY+7rO;FPQ1c zY9T;Q-XWQ$7*+P<=PZAmm(yAjk_Gh#<8QTINii!@K|YmnjP%Kx4S3q3@(gG#f?*k} z8Q0CPYteg5hId%ea>iA_E-D{4FJmzlBgBg`GM4Iw1b zkoUe7e*IHsS6VH_v71QfvoCCki+eh5#ERdyY^rToDnQ7Z@DkQZ{Klv|a80tdC^CEV}DH?&qiZA~j(f~AmC~Y z*JpM-My+!B3~{%|tyB1@Y*+N=tAvsMgrL9B?3R_`L-fJ#!d#ra9=cao%^N!tCU49j zis@^1sN~J z*d{@gQ~3tGL(gZwgk)lQe{@dsp6FL~Sb4JZ;+U8n7hSEtGA}Xp9_3+ZdDt(^PfB`~ zDEkPygrbeOfkQY*F#W;!oP3Q7`+Jj}BnAOTbD!ZS&`tL2T7izH5$>FA9pjS{a~hYZ zA>8iRorHI$8(;48loq6ADHnW!i4poRwW9?mfATEGI*%7yqwzb9q6HhU1$`82ob4D2 zpr&|g1$y@N_6iq&le*#;?#)z+hUaHDLtY}+i&;yeV7Ir|{JKJts_MaI?iAQD4oa18 zSzJf5S|X1J% zXe6eOfPCZoEbm)rHEsNzfG zU(xKV8DE(n3*TT(?L>m!uPGnd%-0MRt$#YFI#89`v~T_*2mWq*G+!Y&x0#3fp85~{SaZV^8oA!`A z*-M``tNm^EHop&N!s2tz5P94B&Dn!0JSuvgA^tq#_?Vle%)v{Ki;RV{V!KbiOzW#6 z`EwfyU$0=!40ff|l>$*=^kQ4JuaAu`jnO(;)`L(s=u%huCUAmP7g?fOXI3A~AlkKB z7R7dmk)(F^@-upAt#VD~Y~4J}3dfQYph+t>WOt|J>okNq8R#iG&XZDNy9|+VFzVf0 zW(&#N;j+R;I>W!*xB3cZJ=<-({<1)9bIyT@_pEfcwPHDvIG{>%1H891mSS^CDyH{y z3F!o7dg@+fXio|`c)OYMsCHWKKz{b?fakKr%_nVXlO(;$6OWvbFHlr=;Ng=>(8CpP zc1Tq4VFni~l)3YPtZd?d0vpGK)V*sZJsv7H52y3fy{Jpo>=v#^suN#iaK7cjIo?+Xtj5P)vY>}~$!zQq2Sf%9h7<|-@wnkX;+vKb+ zy@MT*wFiH~L@t>f>MC~gpB9Y{w|{)A6U=VzXQ9RCDI}t4M8xy%Nt{WvO`P<2sZ%6Yq-j{Wk z5QivS(;c~9N+ee0BIC~(EZ<$t1q@W*xy0z}HcUELv4fw>H4T6`2c6z!+bTmOYs{=t zayyi{=R9?neyDIW2rX=AdmummYOGPkvc#$JwnFrA`NT}zD@JdYpJ^@6b|{7m_c}>+ zIZnO~qFva$c-);tg(;bZXG4R{NA7)5jGlTdaBFzFFG;N>IzaV6XyC0Hs&q?NYxa8o z(CS*7=<_bwle+kI#n3GYldUbB0xImguc`_i=bU@O?LaPg^+!QLj^gz7{z+ zN5dF4_eY~3IUBq!w44w99X`jEwiu)Oks`BHB6p@kLQ*Kg8{I-(q78pFdDv}A9M zAls<{MuByZFO3*I0}Vg6kK__o>1oq16sYmOU06fIe3^$Or*x~CMc*YIesn(u9=d3= z#4;1@%p_X}OL!codyjsj%Lr;~oa*JG$!Jz0T$Ns-3{c`pJthWyzod>Fp@0 zLX#|&H`}&g3V|d-(C9Mmz;INdZV0S=aifu3=c5`6CV`netzNC|1Fy*Y9U~1 zFY_0usIr4l6+8WXM1u`iiHeSqLO%9mZEh4=p$L4ggs{yVYi@F>+pqPUuvM$}Z~D9( z2jEm<>raq@ekRrH$C{nfR7w5^%mvY*-0k%XG<4S6T)EX}ZO!b1UpK3teHP&=&&p}r zK3#()GekeB7(9>$YwUs-DvV}*c9gJl!a|>`goMmLoy&SSe3>!TypdwfQ^dcuDvwbv z=AO>qW{gPR^UU7_08j;b*jAWFa z^zG|84ZTjQ%x?p4t3nD~ow#l@&_l%83dg8t0yK$c+$3XiG1#%h5$n@wc@@H2%T!?F?xDyDk=wR=cTCqDxv z@nqtpr&1W5`#t%dsS65SAQLkW_pmtNeY3wb`f#&$Zt2R#$B32ScCOE;A*aA(WHe zK6&ADne5_Uaxper_3M`off?SK=tN(-DA$J02Tehco6CLEw@dF98#13tkl7%XL{5&y zGx<)-kBxLsiREv6%YoWgJg@0*k4p%$JZ!n3w=O(Yv6=6=5OJ1tX{BGl!LpqWAX z_MI5Hu@odJ zD8Gu9lJl0ir_hsOdRKLjwt!jBs=`}e&92myTez<7>K;qoPw5F-O~XblFE_S|#o3Ga zNdGUSha|}XP*Pg=C!k&kf``*HHB_Ellj5}!6#W%7aF9t|q70b}<1d~s6C^U9ma&o1 zDYdcHs#0AZPLALX+A%O?5*^9ziHRg!pX7CpO@Trj$>(|EyCRgdcYIqE>%?#ux^RL% zn%=+Af(fwi8`~oCQFR(ca#a`+xtK+fwku^|_L$(lHVr@~`IuxsUTH=XagmZALAQk+ zM&P)7QCs@9CENc&OOS`a>W8+^v=%2V)&8VY%@{t$jb$S;@|>@U7j5-f^!06vgM{fI zYu}z2_B$#n?OREBzolXh@f~>iOk$w1z_m%IaPGx=uW4n%l*4vJ`E6JIP_#k)6AYPU z_9Rdh`RAgUWG`tA&5A1P>Lx|D50tz+RX@41A{hcg`?@cGd{{nnax^h9z5Ij?*a@?c z7ed5h$h+Fu&vPsLe?`S5rfr;&B?`pi87BB9C_}@j=z7+Yq>3K24~jA)5xmSgkCJ=Re>u4N ziOgGg(Xq^HRPZPuUgoI-NSPAXz2N+)rh%6~b6`o&6am%Jfxs5E=F~;TIOy}xO9Jp+5^y{j$Qrgifmvnv#JIvI= zU93It2wZT7*`BP=jILUwrnJm*dPhVh4QZWpmbav}mI&R_Uq{L8go^giSt(T6Aw z9-@}huGFiL86cqsj!ita4Qq{r#6MLuM%!b2$2tGCu2fKUwBPn}{bfMMG{;at@vEHR zUVpuaY8%?7kPL|}BK8bx$ijkhpF?3k7IZZ5)ZEIQkA5OtK*f^(!Ib16v7f(LyS^#vG`yk@L)N!tZ&+3REdhb;%m#& z>_<7AS@7!{C-X)`tfnuQPQrBPPp{`+J5p5KLwp;w9jCvhe-?{{gZL?OJB74i zj7xWIo2P{BWXWrrEVP%61`L*zWlbSj7d7;`OvGBw;m^T+C=L9|jtft!XK3CWUa~y6 zX&)T0?XNFvc4RMB`^rPyZoU1>I>zuG_xuXOrCc5{mcsLNMq7M+>KcnLEtan~HO+at zVrkH_NAcG@ee1r?u^q$A-`H;IG#>KXXNn!q=s9J8s8%@9iQig0Oz8v>2($&X)Z`zG zq1se8uk$pj)|Qt*B<2U5zCaM{P$1NlvM~{k7D=?DsD$yt{inEx*yX15o?&}LJeT_0 z{6$??NHWFr9A|zH8E=p4%2{_qO?`4Wi@eti3hU&?cFSF>CW6eI5ekE->!}~-Ov)SM zvvuM%eF+z8K{M1m_sVSi^&60ugmfzA8)NP(aqVMdlK5Yj3yA+r*F?Dhq*XOAP=l@=Chf& zlwzi8x4m|B=X;ykvES$N4LIv0ea3e0SzwZ~@zBcL*Gq9ZW#bkU9X&js7Nx;l&dTY` zd@)UZwih^_LQh_4DQyR7?|uv*#dU#*Xt3LFa%X(=Y`SG-6623m9L+A7Nik)I*<@It zUl4Pm$rXnC)H#}b!e?b_`E{Z;(-J5=ce~VIBLya7RRZDYnGVH0VYVG9&iVZ52imPQ zjmqa46v&g0#GT@*#HNwULOsQ_ed^v?$PlAcZH+;N?yd4#=1n0Ra!(5P55relOtk*J z2lCOPv6#SrCtWp#uDVu5!l%0U9K-5gBcm}pm`=W`QdH3Gl2~g0qG|CI3AAx4YyTZr znogF)Ln>B>jj3iHPaNjuq;PzrnAM%C5TmDtPcvw(iDi zdYVspYBs8IpqupSJb;?803rC>yU)KQKGGhdV8urv&RXC(iIM-X9ZnobGacc{MSne0 zH{^D@{eXebI)ppNC;!#jqXg-R+Ka-gl?>7e3*4bco#LOlQ6#YyrOU#P2|5DU(EYP; zRD;_ow6Lkp{h7(S8ooy#(%W3o3#ipJFED#Ym6KFAQxDZs-peW>Mw;lDV{4^msJ(clxOQ^xj^jRw=RtX2a!a%*1$068jUWG!Mn;(a5QDplnA|B+R&4 zpPSw>hY?>G&oGcGxLuH6Xm27rJ#z~5EU6%IVj3igdV{NU@%$DG^Vl*G5?gPpIc!d# zOUaR^?0fx+`^ow$gTwt(cePb|BHy|Orm}oKmKqFkyYcg$`A{#mO4NtAbO3q}o2WI+ z^rJ^p%P(yNeWyNM?CQ^tdjY%_ zWE-C;D?hHd)}6NdG*ZM0Vn{BZeDmW4mFF|lrw4@Z3g@AZNTk1E%5{2%f}v3bj;iD%_m2G zURSYGG<5?{ThXdGp$f-}>1iM^sl;z$zMD7i_S_d1i#q~+_M9r9GI$zc>%ro)ELcyd zZoe20bKK==ctzkR@zRXX=#Wy(dpI7np^Da6GS&SfyE|~I%0}oRq5Bp)OSflZI`U^( zs~a$yUQ%q*yE+%aV`)m>fz2P8v`C-6ic(PdjgUX;_U^|Q9sQ;#Wm~U1sV88cWJg>4 zE1HFj9Ona$C2}Hu$-`Ip6IRK+;aB4%#GecTMTE2Z1v18(WcQl+Bf=w!VxE0cgtQ77 zmSXdWn|!?`yZY7;{LTbbt`D-x$L1oKgf(U#6oH2(S>D9}Vt5wN`r~G8N#qupt!wz6 zLa!H3ztuf!tWU_xurhkWGffh`Z|2V6^@M=ewv$8(xq#+b~fL1rEh zMmu2>v}p2#Ty%lqH}4rTR4y?_`?BiSf@v*+GW=E?+<88~r#oFEhZ&NEQ)71G1W5Z z#Ate6mwqn^*2ak%NG0TJlplvy%$8jT51J!=@cK|H)DXGGqJH6mMf#ETIk$8qW>{xA z#Lm_aLyZ)8w6b3i zjqV~g?dBA-L^Dj2apacPjiTRAOuhQm$BFS<@TuFYnZPZcR{2Mq9si0LRH)NEaTdBdQYZy z*|^Duzf0rs&iy>Z`z+I8G#%FM5HVCl6}zcIh?2`<`>o-vi9BCfiA>!APEUW4{7A}0 zv`J-F4bMEAU4Ak06G7HqW78bIH-%iWx2nb62SdkTA|2+Z(F78l)IABV{t|jGNm_Q@ zXSLo=Iu0!?OOR}dnMk4&3UO5%Sv`8ba?YK3&GZzv&fTl@`iTYm;1EWUrgOf(yO;=* z29bum0Xy$)*<*$N6ew=>PV(w*hMu`WL8306>-DkOr|4xizUk^S#rBH}F53Ie50c62 zl9juNK+c*YvAG!*_n1#G?XfF&FPEFWEy|P^ya+QQrpgg!0x_8qHi35yG?k4c;%$W{ z6zzW=ZmLKS62Kx-r)vGI9=FuxfvzSN+2=%W*35(^w(qBYIa?JNj7cZK*f-cp&-7t= zZRW*%1m2an)|o<lwc|Elq7JA$<(coL0IicY!k#ZTky?hx#7 z6n>qs$y3Op81A$fqJ_BjH;X*MgPW>iF+MBvJ!o9<JwfO zz>p+fs@3kiNjr~|Q|M81C8mr}?s}B=b!r;DYA}Q*xj(kOX-Mt8$|Yo$A$ggQ`<`wn zyM09%tdQ-)O>si>n@}~X(c0^r zMd8JRMWYqcs%{ny2M+f37+$+b+t}+fQAu-Y%_>Nx%Dzb>3ncmZ{$=V)1pf0fbOgey z%8-i?zs7@_8cuR^A;o%3-O-m4W$UWPIts*Ank^&>&w}V1wKL=pjb z#o;%Pl7v(ZG%Ei5=#At58ISo$Dt<7U@5>^2$Ycbpr}^C*7|es>Bu8kQF}C#pT7WS& zqHA;!pVQl~sZ-}MdwBL^4z>5qc65v!a?+A?VH-%9X4I@APb*)p%nz~Q9oL7D4WY&k zZVplxDkJjRKyHn7(y%DH^Vg&D90Wfj-9FNuK^w|etdjK6G-p{+wWbVHyE00j?%JZ0 zU>tt}(@b4#KrvDwM7nO$ux0fuWeJE2Ef-^|5$%guMTvkK31}VT#+z z@nD}Zvw`5f#3zAC+=fxzJr2iEUf>j?ZgVr1m&eC+k~3DEcV;|`lA5)X^oIqs~$!=55cgVk0KGi z0*4o2>KHwF-nl{#KDvmL2OuxRLTnQ8ZbDwfY~y{HC|Ca!UG#z759$ih{WvWo@qy~$sW^<8iC2Ls7wJw;R+G8m4aD3iZ zzuA8Lz@^=bj3|u;i?NWIDR28@C&_k6Kmn4PAmd{T58I{uy+@x;dH7AuRj;OJ)rL`MG!Egrxh;et!s=fxh2}gQDb16)ih(jsgGl(DHZ{z3U8{yu z1$NBx^OZudPJ(9UDSCopU6Qlqr~PzjZ2kv28-ZtEX9goBB@aqs zuf;)z2=l%uq(7sk5oM4rc%I&mOCCmbyY?3e!{EqMAXkuic%$UELz7$ODQnZXSK0np zO#3mJo5huL_G}8rs}>Xb21$(QE)3e@A!S}wLbg%Q;g=6d3pO|N`Vj9U9r4n?ev=p! z2rGwl^kzqyl;qn;R8u55@wmUp%QRpck0hnCFMRyGo)m`0TPT2on}CbcYVlphqnzr45Mq{oR8#)g2oCX(SmGsZ z3KnV_LEl%ySB6e{Q6R5g8Fb=Mjg|ACs)v_r>3%ht2ZyyF?WfPJFHA9ShZHj*vgR^7 zL_eHMsTxKNZ!8e{4{nLbba4qT&iuNKmtsKvbCF7A<#a+xCOrLN#+HjXB|+7rhLK5} zFOze2=mzrM7hY_5HyS4cpZgQkMi;|4KX>ai9CtYUS<77;WAUZHG-gV$q)j{jpaeN2 zD~POzL>B2~Z(cYzq_9Gp95lAHjGo$x`N^<(ooP&kBAbwV-n{8U_7qWb0>K*H%;6E) zX>VC+{#l1^@tM>J#d2uA@e*NXdRY-hBh9z*mYz$07o55DitDF--Z~lWU-dZ>F!DG_ zcwG%OM2T>Sc2AC1%nfl^s)iMYFP|gnzs7feAy-o;l`2H5q)tt~iU%Wn6TTH5SSZXmi4;fXZ z6M*EZ<9&+O4&Sexdh0{<=#h^;6CJ)Una6NP@yH0M??nKGOv5E*7v)9eh}2Bt{bU|h zXNUSyY3jfSqIcvxt-4BLF?cvLnC)IxxE;w-SvkGl>`J>7 z-_g#8Sk5`wbiI<=ST|iaytEZ8dcn621(h=Y$g2D9*WB{ytUV`wvS9eW3`G2t_)H}y ziiCu@)&5GEy;Lzl%gq&ha?iP`X?Hu~z1?<#;8&u@qx$EYx?;4|Iu1{RV+Nd_uN=00 zsYE4p=7mCo*fW_w`|7DLa&r&t=5N@P@Vz=yNU>H!JfDoPy*6-u3Zma8N{ZijE%3S0 zd%02iCZK>t{mUM;eoV=ZM-Qu4XZy#OB(9V9_dL*)Sz16X{F8(}Ql4d+wjrKM&cWgm zhDn)bw4Dv-O|Tq#gQ_iqYt3Kp+rDDh(=YFX&tpb(Dx8pyV5QXyoo1a%FWn|xw-VP2 z25ZVc^>@6am4NYQzs#t37+B)`fI)U?lf+aZ^+!eM1hcg+J5s5P7PN$e7c<-wtAlhb zx%Yhrrjxpz{~*J5fME9~aHv5Dt&PJ|lwT;_LmBugm_m)f< z2X{AOe>Tv$y%lBq`hDYmn@V2ybCXU&u9YO}`0yt!=xoN>CSS5H=dv{0R@eG>pfdTG zw+V53AEwrrH{7+mgRG}(yw|?7aB#eC?<;=tt8T{m-u)66G~+nUZt&I9q|XuD{T{x} zA8oxfG^7#Nh+Nc%#h3?Iwp@g2X{Mdpprw(becy49psFp^juTjdXNK?dM|WRQNiha~ ztjwUPBH{ueo9xl%F?Kw#U=!^rN(y5c8qNcqsKOccX zg!I<+#*K%m`PNEHfU4SVo~EOr*ibQB6Gl2qvi(?BYtfhw!W{Hf`|MIpb{fnc7q( zZO1bS9_=w)C3d>Z`_3{!GB~68DR^{oHk^YA?EQakvAKmv-GvW02-iM?PIS<+|^Q5MF9?*qj1i*N+s`Y|MOa2pTYY$R zqe*+Qp7H?`)}EGT?&&a)*i+s=B-ya0(K%>9xX$zt=OTy#fx$A^i?$o3Pf&LF#IyXJ zT6M{=gPG3WdvN-t#uCz4;qrVW%F%2n!FyT+KdgN#q{8a-N$3KVprm? zR@Ru|Ay%w4QR4VD`T;}$or!{+Vm+>JNn1bQq;sNQ2o_%=cUNnU3z~r4AXJV%l8F}o zrM(-zzvOFwfEbS?ZX+Qa@f~_33L#`xo^e__QQB&BN@P)tF3{tQ%yE9df^>>M=l-5s zO}ImoKhi@}Eaj^d6n~JPi8w%_hlzO9XUcrTNXaT*`enOpd!WMHp9Tg$ z-Rmg~VXHhm!Vr8w7(e6FJ%khKSxZ;?jx&2n=9QdK_(1)!bdtR4F4lPs)?2bS!)e@n z-ycoupL6857@LhM-9XF=H|Nca8or9DTQa>u3uFn)8xG6LS`146UV%^=zkrqXDM@x7 zi!k&Jfz}%v@C|LMV@xXD2fT6GFDIdRJKte%N^08T30!l2E{!R&GR2H7%lPR}$#eX~ z>NLu8cxr{pvP^``|GEe?4>nYhDp*W9RHCHB&-}?;`VQhNyRkxxUFQ6za(g;mFF}#g zeG7Y3$>e@@KtUg06{Z}AgI0A{hInnRfg{ZH>$wm2PmDRYfhDwPAqbkw{I50Q*hrg9 zKUqYBT&hJmt<46r;4B_fqDTx~AN337y0rRhaRDf$fsnHbD!SCY8!q(7>HTo)gVtIo zW?CZX`$$*x$7oW9S5wsDjrb&Eak@u(qT|`m`2wap_$pA5&G?5ft8qS}Vl7*J|4~!+ zF*}J&1!G=y1Ig?AdCS}tuC#Lku{zed&L`{=ZBEvqk8RhLjcpe(4X9~vld4X?t&wM& z2@V$LX0r6(Qy(4iV)HTlWs zU+WwMvk~5VPE6W_YquzjvG^+g>3{;y9|sg9TwziwFawwg_0y-+AOnuyKwOj>IEi3n z0Q?FG-TlGo*c!e@5Q-j29oory#Y@E~}Nec%@82k?S?v5OwHF%cuE~{YT3Zv%Z{tHdfz`+E- z1B2nb0Av&$Oq{9txZte-IaLdol?fadeuzT(Pd<3y`0!plH8n9c<=}vDad1E(Kr#jr z9|s2~0J*^&96Uf74>wQ-)ZyX$s}9e4*>1I+xguFylr?}aLDsVTX1Ro+16i5 z7yi2>#OR--F;U#r}I+zt#RPR{m-lkoSLU`kx~?cY5cB>z5ZufX?9h`Yk`W%;4hy zJ{|zs!F9&N^@sfaoHcM={Utxh{~*6VrwPzoFvnj4x|1K6 zCJ~t9FO+{LaVHiq$6p%yhvI+J2=Bj7hrfq9AVR3o-E@G!hxFgm04Re3V);GP;X@iO zI=Beod0gG@lCATy9T$lT4&+{6ZC0kQ;HfviC`AX|_f z$N}UCaGle980bEM`5E^KXJVq&(g)Y{Z+V8CLq zaRHJ9)W-qXA27AVCe)ypnqmq{;w);)8j1#1RtClvwsLGT zFas-U5d7D|NYus*z+~eC>?F*@TAiAQ7cf|Eb^s=v2Am}a3p<#t12stVukLd5a{g(< zfZV^P5x@q_1gHxz4>NgYVq|L!5Rx;oF@u={V&a42{c~u{TD8q4n**omh;Uv!_av%2 zKtzsEf)*!-qNXJ2b@c>63cK?CpeXTM@AZCO`37BP(H}j1jXZ2y^B!G8X>*{R3(Q(JMPz=zk|c(p)ePLZ1c{;e)q(bv4e50U# z`q8ECYJl$L*4uaEveiuAOctjv(p?zh-zQx2@nq@<_H^|1z4wJIzklN<>RV2b97}2N zp$`>xSZIY`9(?ahknBCyd%>Bm1L3f@c|=o0teLSnxLuoJ_hRmgNIoHDprsa%WL>8u zbvNG7=tI`TSM+seBSki!Kwcvz0yad_XWxpbax5fjQHTo0$~~H@xvgXu%34 zNb~1>F6Q$}Dgn}INoxGS1;*;MIHA5wxU9O2aq!$QXq->LQcSaibdY9@qa~Q5-r$DA z@Cx7M?eWgY^fuRl1_qfttpMt2zYgzGRxj^(UkhFep9FC#QiwrughT1^%-8Y-#|^aW z`8UGDct%1j_75<}?FMnYClwtQ16LE}&@jiT(9A2#Q%L$Oy1u0t35D~&eUNO=wq4HB zmf3dL#sNO{_pbP0Il)PH6i+dj!F8Q>F9u{PDP!_9@IZM!&=rxSnwICkF+e@b5}h`->W^M z4;d3Dy+Y1Hf))LT5Qg_}XQoXJKO4yj;h^T^gHi+b=>IGIt`FcqIiUX}I4%!2&u?P` z1%s)fP~aMvo$zZ21p2!io`<&wPw@Njaz1YE-}U~k`y2js4~`4Z!wuN)dH{su!P9RV zK>L3~c>A0{U2cFLH@e-{pVz4PNK(dT{OM(&zs>Z2_c2@$Fc1NAa#8aE7rcDD|L$V`iiQ5= zV#Fn+Rb^FKG~q4~(1g4J%v@gP-~0^91Jiz{s`82|BTcAR|f-xuYWoi zFV%0aw}A!7B@2xBopZG0`}8L?i;Tv@DDt0S$&=45rLBwvLT|3eD0-AW$Fj2ykcPw7 z%vtBI+@{!<3+J}qc??b!QBFJRZAo0zZKotB7qbmA&V{{rsZpO~7Kn24y>?YSatAtR zoSa9whv7C7+vAvPHa?j1%x5$_Hv_A|Yh*9<^q1pAWWCR9czR@V*NqNUPra5Mt9wyT zHqGr{Q`t)TFbuKv?xncpuXTHIVOSr zvZK!lGa-;fr*Z%Rci9yY5rJ>H9pk-O@I9n>RBYcU{h-!P_5fCgYgQ-0`}LC}p`{tv zc^?tFxjwAxUqo$(vS|8l%KA1V6N-%Pi=&$&l$%kTm1VFanq%mqi@BnjniuutQ!*fo z6e(1G_f71jQ~j|QkCP=>SFZVA&9SDtpdEB07bSBwt-6!H{IBX#xdBt&dZ@X)NP&do)FHHB{N;ucE8qy3ZEDDkWn{ zaG~_;(jk~KmZxh)5~?L(FSIl%ONfu*Zw^ln+S2J-BfO@F9S) z=i&nlIA8|2Ik^F62G9mVVNPBiYEF1vFi^$=q2>hxb$=Ul02d74!ubH`aRG5E+>t;a zKmzko^8jVsJUl=Gcmr0=za1SIaA;g`X9VvL1n4JF&IRzv2?gkK0sM0V{RjH{yH7xW z;q*9w1b0Soo&h>QN!r~uyJG6AA$D9{fcxU2#B^8u@*-!uVzz;yy|4f2l^xPge4ha2Dspv(oh zdu~p+WZ)RUH6I{lfMaez0}!B>kl(ujzhwoN*Z-A(_WyRQe1LxcOk99G;fWjAZGa~z z7u?wb0RjO3aHM}O5C2-XKmjfO;YfJ^Vg0dgF|l_tu=>+MfB}X5=Vb`6)&g7v&~q>c zpfFxwF#g*z;Q1s0gAwS|G^fAtN3bH7`90P_F8JOdCT{plJ0T+{w< zo`DyLMgH^*)@t8uX7~waw^=47d|^vT*n*+0zVsm?MAjh;bsOW+A9U`O8-L-nJ338p zj9{&nAc95&tb-{c@{?Z(Q}SFoyPl9ANra@|Jk{y$wk@;`RDn4f1+F>#y8bnHIbJ(4 zcBFA&M3E#V96Yl%KYu#+i|_HmY5z>_@kDWC%=6pnV@tHPpc%$nORj5|9jq4UV8X8T z)b>VNTTfx?fH$=g%-#f|Gi3&!wV>f{YZ=(>d`V?=uP&yLB9@@>7YpXsCR z-+!#a+>bJ7jH_mX$6Nd(3~m>&r9d@QyY{_q?!s=Fn_A#crsmh@F!L+d4LvUyP<4c2U4CU&0B=`H& zar%v78D1k>BjDdJ(-c8oV}4(8jW&L-kAu62NRf_9%CuR5CyJrxmuD*F#seHKhX!9T z0ds~{rt~{%4Fr*onwyl*u9Fu>|$utt-{DXztNShSNAnJdu@i+E|y~ zDpu&F)?m!p+6r4+=pa$hs$|^g#O#?25915q;55+4sFvenz~&2iw39u_L{z?(M&C6% z(k_zZ@p7Z^^GCWA5B16F0p7fnDNA0-Ch0YTy4N+S_Zj=x;<7#<7PW!I4%;?A5zwrp31k`zxzNi!OWXD^<&ZIgNkHzAM<@m+0w3N zBXP=Me-$4I!5?p%?x*7QgZGUG&xdH4`KT`C3Kt9;92A25vbbc_`+hUFjSdyH2mv(0xZTHy_Ei7X5xeHG^EB6>%^2LBJ-2U zqtnUA64IVf{YR%Lfk}?i0z5q%o0=~C*gf@u31JHl9E8~wbdJ8I&gyi7btZnXWoVO} z+D=@qI@*&dJmy?f*+;e`lZm ziR_jCFh)SV|GXgr0X9VdlMIIMz48Fw@V^=&z?}cy()*VolF^h9my}~sG;p_t2dZLL z21b^KwgwKyVzySs|AS!y=>K1a3Gj)37^dHw#(($pe=|&6kl%YL|JoCIUvA^affL{& z+~pJ$i-9CQO)n;G@m(52J_|=pw@pY{R$sdt!|igKJ?C}Yy&ny?0#^%5QzHdoP@OT7 zl6ZoF$)Yxnu$sL%hRP|0@=S3H>>!HsoD)l4(yr%K91BDZVrUI^b~IW7b}9KiLoBb} zT}fQ=akMO&jc(KwZRb-yvwRnp9|&r%!E(>{lWkvk?4O0r-K6M%+52933(xkwt=js` zhRoBKZ(KPiP<#6~P+spvJZ83Q+aY-u=VU|KT-SIEv;hq*msJG zYavD^-~~Fx`N_Cm3KSO=_UO4bdn^oWsfs6R|Bt=v0Fa_c*CL`AF=ug64+CK~FDojN zL=+SO1r*c;7Fh`k%aTOEGiNb-Vvc9lXU+-q%o+9c6crUCA|{OQ`~IoFwz{XgXJ#W{ zzjy7_bXWa3R`o1fop*iV-rx58zk@ga>6V*vHvBWU%cV26ADjEky<2{^#bq;I>Gtu= zd52aUvDp>hRo?Kz!fQXd;@IZT9`#o50ArQ?d}``2)Ev1^{*=vnRiVkBNZL;Nr0H5) z9bN@HD#t$msOsSrM2gALnj*($C8`yQ0O$!}@Kz zbDOKX?K^$H_BY&ye|Ky1^VZ#D?aOv)b<^*^Exl`#(d{-L{NbLDmF)F)tG17SaR1$R z{(FPMNmrEqcGZ1f&b@5Z(j_y;^=;ex!k@P|?|N>yEdyn1u^9|46|K?p4kMu77 zzpt+t`29Ygeg4>0d0$QJ-{Z#q9}Kv2gSp=>UUKVi*S#`j%HkI$4SIg7UfY>Y?Tx;$*FQ0vV`%9l(uk#a^ zmalur1=CA<9P?u0=z@NaJbK@D!-~HjJmi+m=Fa~2bN7w8{l6cbUo?BnI!Es}efS%d zFTeTXeRq6S@MMcm798`^I=kQ8>gErQ_<4)r%X+QA_vcJ^IQ` z@4m3fkajP>J@&%#F=vciI-+ClW~Y2IXX~7zJKuWZ!Ilg5o6u(3(Hm~is!jWxU-o%* zt2QULJpRgVEeqzK+q~wswV;i+N;>3TPxaf>IyPZA1Z@+zd?SFRXLtp&sn{CE? zx_H#_o0gw_Tj^oPo%Gs(J4gKRk9Ue*eDaF2VGrjn-|C%`-G805`qRbJDmtxs%E{YJ z7&-Wk&9>Qm;k*f}AF%$z3tzmo#g*SK+`bgo!=G*N=Uo?d{^jS*W{mo9Z0G$;_gMb^ zypOhg@8>N)ICkPWA3Znq+q_5C=(hdsr|j8s+o!+Vc#TVE-j-N@%$~*T9r5IZk*&A< zW9=C?mA-M=ahoied&JE1eqA(S`FH)szrFjCx%>QY+8>wx(D#|`I{xE-t%nYK>))?; z`J?}{f3@gQ`L7?hKkV&p9fP8PZZRAN7r^!%E_N?+p|~w@u0j;Na-owd#Fa1K*f$T*=FX)){Vr|B{}fsefWBD7c19u;B#5LuUhFs^2RV+*;%u8nsK z(LHjVy%2yp;mRw(gS-L&D@H#jP@Vta0n$1ClLX9J#l`3}Kv*6Q%A9YaA9NuFynJ#h zM2E_&gbfNh%u>jk(T%6FdKFV=;F%Y@LGQ&55&FY3&*jI%P92xZ8^=dU8KO5V0<}T^ zpGQLSDdA+m^?prVL9PQYcF(W@9Ie3s(F#-W6gpdIoan(C@Sp@zD9(rQaNY$8LKY?H zQE^Pk78mLht-(@@iqP*GZHYQaNK!AwC}@I4s%WcVSJ^ZetLU1>6liPCi@J037Wu0O zws3QH>9Ko$ZhkS2;tFt-m!H=OuV?Ly{~?FTJRCyp(zo)M3LIl4iaO@z4BcgLUS8*% zJ{999Ps^FssrZQE!lAq7be%9^Z26$_;rmxqC$QDg5nb`nUH0F1VBbFNbH-MTDbMLu zK4MHIChl8aHj1^7bYD=0=Qy+ERm?L^tdV1S&c!~t ztmhFrkNG(VusOGo@bGEz5|U;D}caRQdi;XL58a1AUE zezV>^CJ&>bkouskAZyDHc>)ZDSqNSW$Rlu*KfTX!5#~~V0*82UQ{Z5(=!&p;4|6OH zpxK3i(+hi?FEkS-Wlr9J!-eg+57<1XcJCD_9<#OpG!aG-=E@jwWeoWW?ZEA>msd+L zUu?waR?gS4j`mhKJjtIcf0Yky1iO4mIdM$;a`>>a0u0(HcI1S&LWf`4n7hXM3h@G# z`^>(|f;Gi_PI(ZPpC>*CKCvgv(N`!Gme5Z*ft~`&Pxb8oba^36>YILmPklYZ1C#SH zFH|OK2bjaxJ!CEQ4R{I%k5b2+k1NVw-1TCQ>U&(PW7-I=V%R=qK)jTx=!JYiRwcMs zgfaOmgggo{m+w%>>qoQ`z~g+*B|U^^al2#8l^6tg9{C5*!}XMh$cyML^r^YeyDcG0B9MKL|;0Xl$)VT@&RDfhCYqaV!wzT@}2ZShLUqAr)=pXkMsxli(I4?_iW6}79M0l z8xlLBPf{;{Q5$|GUXeD&_FC$cve6uEV{(=}`C*7%0EWONZMC$G6u7vdb|Sis@+&+O z8^#*zOFnz{!ch+2 zYu7iVtbKT`K8z0Ph2My{t-d3=raTcJBYj0}gmmL_<*wsZNp|vXD(fruB9KoZ{MFW0 zxQCPZWGxSUNli|l#+Y%0@ssN~rcFxijToYJPf%{oS|3rmC4SH&Jfw-=>g&?OGDngR z+Jo!&7+cx_yAb+k-jjF+{vx0~OSp(56du>(xwU~Hcy>B9$w#-WgFglKH-tqQ(GM^P<3Z$> ziM#>Nm6MKFlz8mYV{Hk1i+wOYx&97*Wj^H7X!cM2ksjDIeYP$&MuX%HSHJF-inyFuEE_UZh3sE`?t_ zl(@xMLi>xLPvmRzBJEM)J9rJM-zDb}p7Cpu1#tkM<)3S*FEhu+N!1tT2If52rNlhO zUDsc*&c-ON1$>b^`wOE>!N;*5hswyT^YF?&CzVecd|W5{(3npe?Y^h-LOgCPbntoB zVZD_T`3j)VoliOOd)8|*z@t7O`o{chcuHC8$EW_|$icrRQ{DpC`SAtFk?Uv=k@XX7 z$;wv8_)PI9z_Ibe#39;&U8`fWwn2OzGL6jjv}eqzRgFhIX^em@8!azUUaC8HOnPwH zd}UMDpjuq4=7-J~linJ>)-fOnKl^zmMdZI*IZd zz^SuN8FN8Iops8XLA^sB>W@-K&>pxy9%QpCn@)}HRKK)6)uTN-C9h$9HP=(txG-$I z927-2o=MHf*zeYeO-QGeX%KD3-!!*~EXkiQUNuO40z9hwl$Zy3NsVsuco2jU(MzGUPd>Q5m#I4l-5n9r*p zGgkX@e4X~`$-lyG%=~F`j+MLX51`5BtGI^?jrHkz0<9V29?prizM4kPSJ=a;{RR1# zFQ!_XHFKFW?E7=-O961w=_~HxYWyBz4DWI9ETbN-wmPLg)h6oNmxtt!Nd90u*2Wwj zR@lCrIRux|9QSZFvhc(eN0(XMH}bVP5#ObDcl%4m8^CXp9?ln2)eeLn7<$~pCG{fv z@Yu(Ye1RBa<>d4CxQ8>nnEDItA2RavtK26-UY7lJcaH_O8250=Sm4=z^vO{62q07T zD|}~5Y>{(9@_E`bp2z2KkO6ZBeVe_XIp25f(~XU%`(gIe5IS|_k+SaS0iSwI~PK|dG`~0*SziN^mPGv$GaH%{YuA~g<1MxXr@?9$J z;i6&_1Q(yf!Io4;^gGy9+{1b0;OWDB@1=@*Z1ly+_#7_j@Alno*ks(p)#T0i9L}i~S(l^cV67n+;*`7#sd3BPw{wt3Jhf)f zdaj9kxTHKa?@}h&jF~TRX`Koc)$JS(HKPYV=*d{8q@6ZGUAuXZ_kD2Eo~;5#+`}1N z#_v*kY|ypG$g$5iR=6Gxd&ri4zkk4w;&V9205yp{htrxlAfCi$aL5CQnKpLQ_93UZ zfAh-0(}TtRTWI{DoPzSN#4FksbG%azHrRWVQ9d7^!HK`e{hQYob>B1Y-;7)W{MEik z$-0BJI((_tzl#HW&v_hSx%G@YW`Ba7k@-3_mf%@@CI=ZIKh?RPQ(e?m$IzkYd`{sy zIl$(FOmrNQUn2SS*^%N~Vb5LZcWURL-E-Du&SoNG6xTx_!Y5=JpV1}b3dB+qJMcyK zK32H?4mwsF^~6K^2lN{EcaVYG-)TM0SmX2cxW9w{$NgRM_g@imi2FMyHo89KjuC%W znEo!x=TT#@R*c`VgbbK35G&*JIU@tgBkJ!AxH+V$>L}8nwMr&q(&^MQV6paRImD+m zI%|LUkte6(xvfnY4|Jb}H2C&*xQ@SJua|Qj@)y5vS@+(qzV%CZ)O&nxm)>WMpQ``F zeVsR!s9Yc>PmIapSHQ{7LR(PA9MAMdjg_8@|iGIl%Sc zk$F!3W+m=hUdbE2Yd;{hg8BDevcUro5)b^JE*`5hr5B%^pUwMae!H+R6%P26_qSm` z*96+t!yf|_J6$;iUx$I1>gSs%}2Kg__&`nj@$gy+MbZf`y7vk~He4Jy4&zSl~*;Bx7)HiY{UbK~C+NIayS z$=|XM`WyZcHUw9Ua~R_+-@Ae>iLKG@aL=DVJ^yB1wmFQs3$dEHn(xg?F4H|@;)Cz4 zR5^_DV(mjeM6OjI^~Wgvo2qd>OUBkF&1Y)Und}inSC&Ij=+JtXWvb6rNC3OgXD zMea=yufY{&Z)%gSAG>qooap$flj|Yc8XvBYF#x*zTQX+cbMv*SH>59sZHDQr!QMA= z`2;Q2PH;cYi;w|y66Zz8fjuX4H*7cEzqOBc69Xk7N}z2aHi|3L3> z2G9Oj!w?=n!LOX(A*NVfaUV-+zH=RfBIlr3Cp8e)kq%S4WOeQb7<}KFIHYDLKHZ~I zTJcHx1SiieGdy3E^I6UZU8K+QlnD<;9)i=6ahQyRf6CF&>cPbxmNGK?FImZlxIwS{ z4FJHVZ1~-UuX4%u-|wSsi@r&l(H;39Jf30d{;@tIZx(NmU64;BZ|X~W2HFVss}2AD zJx2R)u(QACdg3**m$PQ9w>YUcACAbsgGsx>{HXQW=z{zNT`qjYk zEuf7&+p$Yyh&>Ru>{}Y%JUppAVXY_sfSzpjnmlWjKEhNuv}1f19dYc&H08n25ZE6A zQ2ZW2Qb!(HQ@*q@+8XtYi}vQ~SjH+_qZ9X@9TQf%wa}f-M+T;6J?N0{)K`FBHE!EA zfMLhLAD@{(1|~0BIYQ<#hI|_Pw^)p&w22k+%mg-OZO3m%keSqC^bM6?h)ssUU|y=v zy(pk{)Vd`u@B&-ZKm)FeaPNR1h2MkRv1Be$uH?9{t!EH>By94Kki!d4gN%Llt1W~ zTAq3Y&3Zo@`D3YFR~vYO3$&!v1f<9KlD=m^8eQ7lwG9$)NSEq3h(`7ui`ou8v(}|6S9c7!8QTx>!qzgr5OZkrwYDF|E#MT}G4YvkWtFlYD3WY0($QJ9;}-Z<@*@K`S~X32N>pS1JxNQr$LvC$Ikc1=`8AR8xv^TL2$sM)Mwy_yhhe8kV^_plsVSP{l<(tPnl>;B(ADw$G)mp{8zivXM9sir-L{ zuJ5Fb8TWAM{WuR|zX7&Gx$#|{jQ>p`)>ct5E6#(-ed!F&`{KS1wq#-&e;1{`8RtQh z_f1c3_c19e`WJd9soPmYu_rTo2_ipC@YjeUV>dnmj+Hg;XZ_xe)}ulBD5|dmKcqi$ zUntIJaXw?M6kx{<*4IVF9{L+{p6rc*uDGwO(M`NhY2s*rt@8b88)NYEL$}P@~HMtIYP9bdPnx1yqL$dcZ3YA%>3~s&Uf)SB63-yy(2>(c5Todv;Kr^E@yP3Xn`b_8SDZ)>36^2xiaJOf0$;`yJ%Y<@RqgM44g(uLsowz?2fj^vqaQh#FO#g>fjpAS2~QW?B8o79?pRw`{%gtjLVz=I3jcE#ict0o~LdM zp-W}O{i|$X>$)&H^||beWk9RW57YtpjenN{yJ74@@4HgIk4w8}U#@ircp*O-Ur{|! zRE@%ZoELCOy#recv$gm<7q-Qiic4e!p~dI9upQVJ-#wAvqM*-YjEa1(fqn{`i{In* z*%JE>^<&x=)D-nzqhvsr(jC_*&?WO2 za+t)Rh8)vBajkTFBe18){rhxh2S&D5#;$!r-jVzNajj8fpT>Svj`8_;9rpjVm!xg` z?J=%3;{6cD>3Bb+CNHwS3D{rp)a)TOkQ=Y!{gAlU;C_qP1oN3M4oeLdv7Zq6o&tLz z#BTY0GS6YyUoyV)UM83JQ9(H&-Vd>|RDY&SQ^vUI_A_yfQimEP$`2s>ct0fBOL_Lz zR?__t{;q>@%l~QPBw|uSpV`Ro!MIlmS|m@ymo0u1TkUty!#Vo5CVB26|E7X3wx{e( zVGYlH%;}5MK#H;9$&iDbi!{5x>S~#u&PmvEki?|OB z9m)PZaM*tv0ngmHO_^jnM__|5m(EQ2ap#d%KffN@C|~g5pOZ~(DK!FQEA>GLJf*2Y z_sEDl$X0kiKn_Y%p#6fM_)HHvWp0Fxuaswcj9Iix`Vnhb__C+Rf}dI&VBB*3O=AS+ zh+T6nFU*V2>AdkRu1o7smm)S<8}!+u_#Es|@(t`*=D`3{`aCyB`CuUI)oX1l`P0Nk z$(s>$NnS=>QBvRU8-uAZnMvM)%+l5bkdvH$vK{~rQX^5;9INjeS&Pr(K4XW}N?gPB zoX^(09B<|9}xHFRq7N<#+mk?b4J2!_Nn8gK`7Cwl={R zu|2@@ug_F|f$RLZT|1>+2H?XjD=+C8XlJ;YtsLoRQ8sRKwc%ZK1DdQ%I0h|(7cvvQ z(hm(St}flR4HB0S$E}^aG?I4EBltmUqy5bRBOA-Nt5?uen{!iE5?vuJ`%5ea7{SQ?w5^R`WhCJCF2H=ZXWL{yHSg4GNE0R7*kIDeF zdum@3dm?SMsrZSa8_y)pa}R}k8j!c-tEP~#Ker)OG9NINuV~zMYU^8zkuR(nP*C=exbKwz|z{#LK6&S3afs=8O`WCsm zG-7P!NPa1M+A{g#m}~JJ9};6JSIY}+i1j;QTN;Uj>$whbC`BH)$7hm*IF~R8llZVs z>i)Xmrd$YN;7jeChDIxA)Bjog;0faL z<{P6ot8elR8pKYBCqPG%^L(~teTFjP8e`M)H{A`r0uwUv&;wldJJ!->RmUP@r}ohM zDLz2>v{5X!u*j4%X&f{a_yc7kJtE)U(1fx|V#Tc@6aW;S;83J#->D91kM*cF3!5 zZKxgACNFXgWa6<`@&}nP7cfs+e6(}kC++4l-_)ko{!vHBJ(EAA{$YHU6FtAZfi)aqi(FtQUA0r=&V+}nHp5& zPWciJLFpoUoe-=&%&7S9JUpJ0 z1p)nSYfIR^JrAI*s7yGfeCa3jHOv$FM#+|IAYYHY`QfEo4_uM!@my?-I&)(Zar2_h z!zSW-6!L+L)F0z|6uIBn7Hfcv|2;L@lN&#M_UzUs(r05n?}@K4W@Em_>`=RnfUmho z^9*r#;DF(lx2*RI7=y9U*6Y;2Ur*V_b_ zM-c1Ut7{|i9$rJu_y`tsvXA#*>#zsw zt2I@ow9h!7$N5}pPwrc5E~W3eHiPf9m7vSUKV3`TcE_|a=3&G?;z@Vj==yvh9K?(` zpC@(kx6S9wrL;-df{kTKxdZpCt#h5`l>itkOAl_}GX)Q_Z>;`ha;M90 zgViFy?c%Xx_>V^^2I(F3K6)5FX5H&I1l%D$=M7(?L4lf z9hf;b{;M4rSk?~soolkFMWmj|)C5IQoIm7jiD7la?WlUg_gA08+5%ja0{zVjw}cloAG`Sd1o#hk#s9mp<| zJ)#sCv=OyI;A^zAT=Jp(xnt6U%bxKfo=CiDh<603FP8?DzoAEB2IfdSWNd;?#GlRC zn6-1YHLl}@xhZm1nB<4L#~Oj>^Hq19zLD z3!jaE zn{qCw_chm&Pt~E}Ih}1t9ya+=btE)_GN9}`j`eYmVz7z3K=4!XJBf=m~@D^#u@P){l9I$8`R-Zn*m6*jj zz^Cy=>YG)`Zj61X-FWtBGs#C>-xPijMK_*F9*z6f%=*@-*b3cBzhZ15%vU1o0Q!si z5#hm?=|^F(8muP-9ct$(F^D!3kRuwMM$i#OBjqOYiTl=+{zH0n)~m=3DSKATBd|fM zPsViEqT5e~_G%j67;}QwX>Q|l;!!&bnV)Gd=F*l;A4WMeSl4xN3INDWGX8^?rh{GJ_WtfufSf?E~sbPk<&|vz2LsJp#WWn`Y!Df{KkDNWB?(H zY?)I~qebO{bbVpmw>tSk;|^@p#-OGeGxp)qy$$gz?rYGe-289oW2|%i%Jn1ZSNWW= zgEf6by%O0=!2hHV)qaApj6MvSBpv~-^rQSvx^Oe)zQ!y*$7Shc+`zNQeuDA{s4*gC zgc^~fAT+0Z6bf5tYfPI*7*;yeIQS^D)DX+yLZwI7a2i^&6GryNJw5uS(P zO=~|`YwKvnP8bej=PG4C&`an$Lx$(bJwC37Kt#r{kw)8>iJ}|NB9^|p-mXl)C1a&!~F`U&?*X ziLk}AHi7j)xhqmWKD$#JTw(V%pr^Pu^~MU-8RJHs$40&yYbEZ&fIMe8j9v6O=3UsF z;N&_To7mQ1wUNeTgU96=E?46Nq}$ZKjZUMh3!jf=-LsiJ3(A}PNnfgQf^rA^CjA>1 zh-tzTWlx!dCVw5Kzs-^JG|o|5QToAy2M`&GnY0=aT9Kae@lyeC_j{w<%w|8@nHC} zXDuL5dX*3vi5^rh$|LzDF7^*xXY48~`B1KaA-N6kC?kG%c@Y{ZXQKnr3CAuS7*occ zb+jMKle8JXB_D*xI#qwSq|c}ayFLJ(@JVUc4nfQaN2g(swrZB{wsB5ed zIZ+mnuU(%lJlZ7X58eo`ws4B58}bT1gU&Zocnq*TzkLZUpp87+u}fo!J&0_Ky^%K$ zPijwCYx^>eK~FZn#kcz~VQ_$#ek3~L*o|q*gIOC03k0oDy&~zMJ;T<>OCj|Pz9fD) z?+Xae>f=VPS??Q|b)q*TBRi&^h@1L_97VoSvP}moYOThOAR3fM%Ehlwti$bOf1~yn z03%(0A-#%s6_?$08P_u!%V{^zPm|W<(2>|iEowZe zo4B5O1|L#4fG4Xz@}~0RbLRXEYhWu^tjGTXFBgx{LLLy`Xjes`$v%gz+gt@;Qj0(b zqAz@pti@dWFl#Ms;K`1GGn+gtHI>p}auV&#g;&V@M492f#vJ=R2sdbrb9d5Kvd-PK z5y;Q8M+<%1-i|g7eaYT!$o){=WRv%a!@w6>ur_M1FS0%(9NG!us_ZFZ9Qck6d?xfh zWkMg|TEERj)e@8^eUP|~Y=s8$Ls=3hVUkArxBtG2%N=8GTz^0Yas840{S@P~VfHIA zm9drnsdf%yOsR|ElVLEhG^-lM<{{d?&#sv(rOyT}nwz5NYLJ?Zz8=LdXw0}CB0uO; z#9t{VpNwtJCco^>fLHPtNKUxp+jiGT+cf2YQG!0$)D6Fm>c@r1t$(iuJ^@VkXI9Dfk)1k zKqEg@ZZ3Y>w?A(PjC5;)@J6h`Bhi`IGG*`d+msvcTRtgopWNgb=A`%u{F-{DzS8~e z!4w#j32@7O;A^yAjeID7?wIu8vOS!MMN;=U@)EyL`J44w*ErObO9S9o`QX0j33II9 zKqq3el())^^y0oUzV9H%jlJ^(#?Vls8$i+_>+ ziuoDW*Tly&HG|Xc6DdA}+v4MQtXXOI8^I@eA)kzUlr?ja=tX%``5DA{s%>@={lu66x1;JcL%Xv_L6 zPUT3reppd8E$2g*A`jh%1Aczb=z_i5Gw2}PQTd%~aaq27bf>G+G@oZ&Pm?}J?@eBt zff(AG@=toK?csi${~!a#7RZ7<0nb57g8p76?bnfGiv2_9wg!Zs#Q85NFXnyge~_Q- z#n1-Xx0$+2;vvs?_?@|2eU$UDUTv51mwdM>87-HY4!gtv>i1S}EUeR{A zr-GbAxrqGywn7;iIflwNn|&F=s!RR@%@RMD{{Tm>O!FV-X^m(2vOM8ghjQ8ipsfQSmyfzf z-bbMg>O9pcaT6EuV696Ve-A~NmPXE_?fB*aR1!A^T0hynV8X|9d50APnA~aTBy8d}bHp||j@r`%kCtYI1>Aqzy=KVT_P3B_zETpf4qO}=*`tZeUG67&$$Y|)1Srn;s36^sO^R6 ztU=x>qz&S-bZE>lu?sHYiJS5MG2}&^GOlQ><#)(SjL_r>IgF`e;?Jq@mOxi1nH@~XhmcnEy1J?Vad)*_lq z>>SXO(l-$nZP>tOPZ)`V`{i6i9r7~0ng<8@_U$7`ZHM`G9>(+w`yFd!jL*QlXy{7U z_gWZ~J7`O>FWxu0vd@SUv{?PCzBK;hq32v1FiJp=&_X_4zR0^D2V@qyj{rE9X3)T# zLBI6$NUm(s;I^VBgVjC)d^3VAJ@=|#69*UnY$Vd9U5L~)8P@nXF;3a(; z3uL}u)}+mqk;;PeR6pRO(cVubJ+9nzOnPwHTmadN&C}NCGvpV%3eV87l?~S0{SVq5 z*QKj-fX73F%EZu?)q7&lT_miiI*0gZ18V>L?x#h01rm=BaW46wZh6MedI`MCd;VCH z@u%8A0PeWghb=Kak=nM1#wI{CAvC(gX)k$jS0d@ql zY?@vlylZUq$8n7jah;Tuk+pB=Piq;mCC1aZPD=43n}0&`eo76)IB$IPZ&4?~_p}e7 z|EH6eK9B1pM^>)fbj+B}_|KXcF*l>%A{o2Ox=uoE;i*ReNA<*9lU1FB1zcm-`(bJR zz$Aaps`GvjY*BE>byAJ|U0tLbQzy7ockw&*b$F*x>v{U5-v^)vskw1q>O0u0r*84p za)Esv;Q~&^IR^8kw}$@-ZqlGNF7?hDiZvyBFvJ7h8x>pNbJB{n(uZ=6TPxG=wD&Y~ z>$-m;c(`YvHk&D2*72lYWsC1%Gr7+K801l4lUK99VR@tsxX$cVC=HZ}ox|q=I7B|g zOB_+}iSRk`5(e=Ao`vn|o%2|81YCALX$jDoysv=uQjfEL;8W_sv*%#9Y;vtb@?KyJf0-okP4b z7w|Rzd3fg<@(6nodnA6$m30_P{}1@&RrJUCkcGlXi9LKy84(`V`(r@hI^rWP(n!4# zhM~jKL>f2;6H@ZJqgzW4=V89A0nESEzLe3mm8oC8(4Xj>_GA1eRJQ40Md<}PpncPZ zWN*pnmpab)QXO)wpAW(fvkCCUy{s@9XgmsAulY&*iM|Uwlofr$o*4$vr1)%%X(~G; zUR*9cw${ge@*_C{erjc)vTjU&F?2@KMmd8fwOLCq^r8IGHX3A43cfj zh(4wABTqis+&G_N$Akkvk^Dk0(x4-9eY(LMo-cEs2l z3MUPH8o$HVtG!Cxbo@bm&9z$#12|<2xioo?K>4})(J^h7m+?#SO^!8gn04vo163q* z3iYA!1aMbcxulVo)|60j3g4DzJoD!&`;71*A2|<4%yIAAG1sdes3VP)T+0i3lD->b zyT%=3z20x~o`Li_^wfGxIpC@{`9r>*Jmr&LEdQi@tiDzx`O~K81Gv=AJ-T%4Ew=xF z__W%(OBSG%J9gyu-jU5Rpiy~R!>efj+i`x)9oUGo@o9bu^~2@{v< zfVP^p4#6bpc>zOe8GgqY{|nF4J{*c~rHdJ~L-LFIlyyfW&yYuCZQ{m;y6TsDWSn7~ zuB(1o=L3$vhm7QPWz(V~y-DGA4r$*~`T}mwv9Xy(s3{ zXMmT@p01wz3UA!!1WlqF@=9I+&%#zdaF4wi`C=}j-OAr6;CG#)yx~*iN4lu<2KpNX zA|t{_{1h2dZqymRlRiHmAghKi(x&!;2a)^Dz#u${J$2zNz%MCF4{qS{$c{RojZg-p zg%{=Q%GbwVlzc6$5IKph0=9*LvGoDkKji}%SU8}^9m`n@Wkla0FSHk>pEv_#ug_Wk zd1T@0A%tFCo6&D=Xw!b1PY2hPEoIO7ln?HU{c{|(59;bqVL>iUQZE2MZOF<@$E1h4 zGIeZ5HIIR1&);Yp#=bqir+&!!dMRG>-K8h1_eZ%N^7Zh9dmeaEy5V}z9l0LQ1s-MO z=Y_bzv&~Tf{7Bry+AtVd?{gD}Bflo^nJ5?K544dUYe%?GU!l+2n8EqDC-MLvja@58 z(q?4aWHl)8TibQVkdw%f@rypD@}n#RY%&Z6bBE32)QhQY!feylm6VH*o{V!M)-eyT zUXhq-;%B-VR`_MD1^ztyK&%VEh)c8N8QK{9QRt%GF&^R5&LbWEngulJJ7|_y$`o_8 z4oO>kt1p^$wtgZFz^8h%bbuc}E|=HHT2XbNxTr(NCZxv4_pmxe>uVY6_QX%h z8uFu`(mykXOSw|Fga?@ucFH+E;9CBJ=KEqS*z@jNIQ+VJX#4)$D{ED+0k~vM5I)}; z^$oFE%unG-X#_&zw0pV^ttl}4cqtdiUDji4eJlb-rab#`*P3teCibqx%j@LQ?++@y^UkkKi8Q!8YT;QS4Z*sPL z0R5>i#`Yf&KWO{3dt54~tn7bPmcOxcl?&}BG)Ho-$`F3iBpEh>V_DCoDA(bCt;&S1ntvvvQb(!ur$ld_p;eX+|*@r{%t#omszI95SP^Z)h zin==K7qyFhccm`+1wCQ|(3#fZvgfcW>DTDdwL|sOjP~ZaUS$lqG+GRbq8ra_&xbuG z;jM}3v@#8%jj@L|#U4iE8tK(BpW|X&YmoO5fJ^nC5(A-IcYn^x(nniVEhIgf>GMP{ zrcbLCEXpq+=7zy)5Sg*|)Bein#>x+R%xb@y@`XGoV^5wZj}V;8SH^qKfTg+1?x(1} zj9poM@jH0XeHPlO=llfg{BxyOH#$(-MX!DwkcFk6I9y)2-$i@&pVtK7<2rXQHgta! z^ZfWc>wwQbC!PTM-1(XAf2OR-0FT|PFn*m4Pbq8t_+X!wPe<;|UApI%DQ|vUvd)h$ zK#pn`*_<&5&7jvGgJjQ6;|F{~?7?Si?%D?PdBqt-W4wQx%was6p`XJ>;{97A1Ih#O zQerD(nd679-CCd3I0nCJw0ep*t$v)zn0W)2)-@nGyyw9)#z@Pb&yQq3k~!7I4;rj& zxZW3Y>@(7eh0IZsljEMwD@RwR>BcLPPN`SeiR}s1CwSjy8(Qmx*;iQph_qQB4z;%x zwof;@bLmvyjQcomEKwPG;!T#mA+Zkz(@{6?pRaySV&RzQn&5@j|F$A{Q-AHzr69RvF>mN|!6^Td$M}E-M?KoYKYw$kOgp*Hy1r zFS-W}T7$E;T&47C^r!Ns{i%<}d)1B|Iz9@K)!1jcvCltN$i3bA)&h$uDS>9+dqk->Fgg(iVKVL*!fE`_3A-C@bAdQM`r*OFzFe zzBR^Pwb8x2U&T11@^Rk_1f40gVV%3Dn$7z;o^_hzRR-$2Dgz&$O!q!r9I_?@Jn94b z4p25crL6Vi)1KVn%fBX5-U8S8@de0{>!{~!&KEQXrS!pS&%Qhy7C%@c)cd=c?DeqG z6iOrQ8FS*jTgZTZP8~pA8dt(}#aa($8ZKArobVV&Az#_Y2W&mp3x%sbz*vpT>J7N! z-;8?Y;mRiCF=GoZtrel7+V4dg8?e6l^Uo8EMu$XzW7!FX=gW zHWmR#><=))V8wmlDqs(c^Q=Ko57qfMNg;VF(pP-8t2VvD?hDYT)F(iD+&?;T$jxiD z9UJ*-oL%AiN5)Z&uO_Cd&4Tpgzc0wVjc3~HFo%c5P+R*4!2-Zc`zpbg$?qbxPN1BD zQ)&bDOsp~Jzi?K0&jq|leTR9x=hDwLf{%Qvo((*$f2cp!Fz!0btC?%%20HARGOSB& zNjWME&@KIj;fsBXOOM6LH7Xa5l^)WmyaQ&F?=_PiT-G;OZ-TzG=j5Q>_QlY1Ms+IW zk@mcT_z4g4728Q8U*?#oo+4enp?i{P`BnWXLe;E?M`xe$y28 zXXuQi&B#{uALdsL(wl-l{Lkvu`m3RdF-dF*-$kb89f>d+1ASj%0nw@)gX{Enj2WkS z!A2P?{uH#&o%-__Xr>;WHYV$sI40*-5Fd35=mu%ld zJHqAqRGbUF@>peZA#)OPSjHFpM)t5-{{`7eR6gW<_@LxMQvKyU)ZJ=eH^LqwvMJkA!UK zU+C4uw-Ea?+LxqEvCh^be)%?5AIY4>n94rV>}!&)j4$?qSRd!j#?MD8chA`!6eo5V z@4w1E9&1232g1KK$1(SLrPf0qCHp3jm&!3E#>IIvc^(q?oLQaa@%`(3*juPw#ksRa zHY;=PWGulYxs$ad^qz6g$#rqwlscoa`#6<*oHvmhX~XbA=Ex$}oUmh_f2GZv-17*t z-=@l&{JmML9MOnvpQ!tkI z1ULt|(ar(fls!d14TcsuZ>GF`I20}(dEm0YkY^d>V*58`fQxX*tEG{A0EWE#j)(61 zFT}xECVXEr%Koffq(#ak~ zzT=*ynR3NE;um@ew>Iam>~BabE~))Mv)%VId|0}aUe1LK<9B(IaBc=ufmk+6gYPXU?T=)t<~cIm6^V*tNVj3fPu6KV14n7#!e~ z^B2M)efS;<2egI4F}%^%!q&(8!O*FRFIJAwi;N++#;zl*&C~uthle)W8E8^_wDkJy zu)*FdLTn4QBhpI$pued6LVP3)2A0xE$X*j5@XDK!nZnr zJ!eych<$-xTZ;fD?^$>l`)RFhju%{n&l-d{Ifk6bv&BdHjr_uDFR2x|R%;!MJ#`CU z)V022%+Q!1bTgg-kB3h3VRTL&q)#F3x|Y28a!DcUCgq#!NjugGF8UMJN8yOF3CU|| z)~BoYGS(60L0+jZ!;jRLyhlD*_rd6ce5~GRAYS?#We>c>W$^*Gl_TX#nhecizx>V^ zfloV6c~O|ukH!(KwetYa?-L>evo2Ccz+vGSeD3dCv-zE>XPwW^z*Zz4)>;1d(=%F= z0Uq^B(Ff*7@g#dO0WwK}5rh}=Ce7?gSUdXV>W7gj&w=ZL@Eg2h_r?|_euE#0-^{(Z zFE)ww5*PTL_Gs=~nrTx$oyjx6vh_12*!N^Gx6$e-(x9}tW731m)>q7dlmlZTW1!(x z?ZEN_SRxb7RiC6xAR~?2u3YRo;@0@)%EizmIR*5Dt<^dzEq#DzVSq*%uL$xsd|G*^ zPJDU*$w?j1=HMf=H+PNS@9i_fhkWIt4Aj50ui|sw2LkEqs7Ie)*k@*LQ@wK|^$hDS z&?>e@JHvU}5 zAme1c|IG~Gk#jA`M&hwAe!&EieHQMEOx;`%Kx0&HwLKsCpl(zj0dOSVn%K+yV|xmf ziCJUu@jGa2%HM|8XAc7Ghg7r#{A@X7e2vZ>X+IP?iHDVK{!V?j!sWiLY- zp}TZ{``YF>W4jh6aBF==J|xD1_jL7yU5_=ER<$3%T$$_GtoOz#Q}CQ36LY`OewPG$ z@VrmP@9^!6uiEBx*r3#x% zvC`&2<|Ej?-Lq%>pkHLs7i#=Ysg>efXmcAx96e_I!KJc{b0PNaX@ig(^Ipd9Kht*I zn#>)ues8K=2)fuOao>!4XrgE8oH@tblk?Lw`KiA5OYI&n>EIb|hVSmm9xvx}uNQoI z_{p-DjD&^wBsCyxLH2!s&*Gv?Aw##f7a6P0jeJvjHp1dM1Dku1kvR0-9^&H#9r=0I zJ>G2JD^q{>*evaoGBk9h+q+dAcx($Y0E>3-*`+HICv6mK<9%MpAATverMBnyUG-zi zniupX?-+BB6T!&(q?v%M@XpA6!CUf2L*i5v4ZmO?L$c1A+I<65Ejc|l*O9YJOim+4a~zaU$N zAB4fE@4A4wlrdNG0%aRPSA(4s!-hin1x<`Qg4>_Zd@(>`AoD!#(l4*u-bzZ$@>J(g)fkKN26uejF+z;^Z1 zj1=0ij`^1{opOzO|Ix#ff1NKDgDkHMeDf)D|5vUztm)SsXSv1i(li9_VY>RQKm&RpJLKCf~$cr3l(KbxA; zt^*v)19T)~z-qMIL_Sn^?wJ0H%jPU2^R#EefMw$iY(#Wr;)>bV(5_= zPy1z@U`(Se0JioWp>U&gMSe&hUpqr*SrHz1nM9*(qGgenqkcqX*&7 z9s6tqB-j2OS?Iym_R2ePsZ0!B`gNF`^U+62U!b}SSf5T;@jI^2A>_!MS7&|dIb2=z z2_L2ncfhrpjLVQ&+`~Zz$RD1+fu;F` zy7T$G-MeL-5chD&9uzW)dpN`$M-I#-(zC-(;vOzM21%VAkcS&JP9ko5{M_&JS)2=` z#9HQvrpQg;0jO(Yzm7={E}OIB9Idd${oaOM0(5*8Fmog5D#_N7Lyl z?%`^DKC;I_obc~$x-qR@^vScey7c47t#R)L8Sq^Io|8kNZoOf1XH(=IlV8+k;&*CL z|FOS-UuFGHjp~oN+@G89EbiSP1Im{9h5LN$ceTz5kXhWjq4slSL|=uR;@-_G2RD|+ zy_?fR_+ros)4RcrHQ)H-WPJXXjEV92o7YC--Ysbtk+nY0S$H95$opuhIi&t)&du~< zZsxk6d<+=2RwWK+p5nE?o#y&Xaq2w4b79&sYJ|97G&*tXB^|Raz@>Xc5M%AnaMgww zqxnuFb&PRVx~#1Y5fAVS-dfs^*q>vUS=kV5I_^`A?Zka*O)OGBqfMq8V}k38GKM{R z-j6eIn(T}*!aw6P#$>EZ=~Ee(LgGr?r^4prJ~er^8X<>_`c&?l)6R6imGK1eLVJD> z3|~Dav57gG^%TyDWS@nwB4DhD?~exZ%9;ke$ooIw&)rAW`{JMEH9!ZUz8jMBU0Ckg z27B*A_08OX%i;vzaZe1HQ2&sT*ar2_b5MMzKZNN@{K(DMez~e&#qXr1uWM+Rt}GjJ zOdrNoZ~Y8tl~`!{8S7hqd+^0mcV2v^nT$)dwt@QADPuhHf%J;ZbsRI-(N43`E8#GP z&^AyPXZ<%^Bk-o(vjE<>--Ha}bJ5zzR-a$3ko_k7Gt@T`o2?JPm-Kf^GM~KBHyyv0 z{UPVOeFivL_hIIpWcGR>G5I??^=INn9I5ZWC7?Bn)UUwr@@aW9xHW!|HmQ4X-(T60ykDsic=yQm99{j*#zjGZBOTCTWSI(}v&e4(B8sSim#G!GQ zxRnmvll?=$;CtS9l>A%Sc<%3Uto{tm_40QX((z#AA$*cIBjYd`3IE_jY4ykp`9szj zyUI#F#7$X&UXd5|>BJK^>TfjQP1pFUlM8wKQZ4k9^Zsz#HM!7ETdmOJ2d8 z#$J&&7kg%vX+9_q? zTkEc)FS_6D7<{SU(2uN7!Ea>_)_Gze=?J2U^AtZ|Hrj7tNRQg5J0?B2wD-#BcSmM@ z*8RDQmt$NW`O?OyU+Nhb;fKmL9jr8K^{mU{_tOG8LUDsm+e4@>1NhN=9tKBziuP!H zopwpS#l94O09>V+G0VbMIvUfajP1DeSUaLkkvGO6Tq^>dk+d0mQv0*?LLbT>?X^kv z1EEu~CEypCnmimvV_@7d{AnE%7Dqk25+8FC^$47y^@qke+5=@|as_3naZ=*9;Y0GL z+AG)4=CMZV9l*k{#$HGRd|BYI-f(RR>nyJZx8x1R5My`Q)MT=jxdIn;Ep!5|l{0z6 zclA;6+YJ9Un3XHm<9~sdi$`c750oM17rDM3#J3|q(Gz80^hSEq$wBmM@-5eSXrWGw zJR<)t1lEx^cdg|c>+KkGvf1O7`bud4pQMjGx$p{U8>Aoig>SVJ;7!Rh)G=XDpJsl# z_iQXK#dwZ&@AR0lR=#+W{ zovwamUl4Q&9IjP=2!Ipkc<9;0S}RYL7yP`@>Raj&*9x2CVVCxNHo%VXxM^}cWd@%T zJ0=aPJL4C!KLUEC4!}K6FNgb0nd>Ddu|8o8XMW%(x_j9svC{>F}bOY(0uU`-+VCizVm z#;$GIF=G$?jqm|ae6F$jUVnc;{{&y8g*ugg&z@tY3-`df&5`uys9YrLOx}`t%vbJS zFW{!^HTr2VwAeTV*|@&1a4AE3pY)n|MSSGNKBGSaru|MD$TQ}P4#=Oz9_k(QwbpXa zjYMaRL)hJN0+vLSrhfw7x9=VbB#Sfo>OGr!|<;nz$5PMaogxWwLJFSfR$ zY;a$2^0_;P4C9;(nW#-tK02n{aH&l-=sq-T(%MCkj$p)D{{wG^`RiR^b56h zwM#Qk>IBF_=pro8AiXT-2f>0rMZ%)(t84?-t6zq}i1(%)A9m$f+p)24jSJkn*ZxI% zE42sCgU)3haMikh4#4O7mf&V=1`fvf(EYP)=?TIM-o&4P&&?I2$J$>I&q1)TM&5mJ zdpBL1LhD{RiGdZ>W6N`P8CN;7eC!crRU=04nzPql>y(clSvmaZbxKQ1a&mJx+GCHL zM7O>Z-}*If)Tvxa$J! z6ap}1Vrj@kiy3J8?1xE-K8m?895Fpvl9lr%!-p2I9XE-oVO($5&utIoB4*T6vU{5Y#oQ z195z8jlmMvEr_@d-73e9456^F4ipweQ%KNCVc)Xqs)}h8N@2%>f`a@)KFrPOQ1i8@ zV@XL~enEatQDMj8!o2+4V#=*wS#@=J)p*J$Cm#YET_#eb2XwC-TUj+=LfHt&uSfZm ziV@}gdv)XO%BqnRKjG6TDE(>GPz~;hAIN`jcc01QM^Jw`0xcJhI~Ej`^eZtq1G7kwMH9A|_^ zDbCF0RnRfNIKMQnFek6HxMN)YC{glq^74y1qQ5ICq1ygV@^X_$ zSB9$UoRa)bu<)`8`<7Q6HM%;dxVRKwN0(I%D6h`hCDAp}J<%(%Z=z4)fW(1`eu@5x z0f~W$LlT1$Wr-1q5%9k8`0u!JWr>lA^2DgbsEWj>%E?uUqY|SNqo+?8T|PchkrOiWHpO-xHn-&JE( zbxw!;qGGt$*s`N0U>x_NhP;MF6FR^iQri zpllqXM7Qqy_wL=V-NF3_^(z}Y7LlZ~kA%kF)y$&&Z^ekNhf`i01nV5 zotCf}<*OG4{de5JC&$m-d&5VUZT4RG>lQ7Xw#9mf@9@ILYwo!G)wZW!d-I@e7VO%8y_TPJ z|MRhBqZUm3@4)|i?duVvKA3y|3m+Ui+-(|M?dsa*MH5p zt*F^o_r3h^j@Mmz{7p9wdgF|%y1%jDgtxEyso$oT^n0ss(XX3zxn;-c$9??GZxz+g z_MNf#`1#%XUirnR72AF=2}`s&M`qIXiZ2apxL4 zo&0fW$?I!hdg#c*?mV&odHZa7W5JqzHdt+|n?_!8>T&Ds`G1$q7`gb|tB=Ur;>f;x zH~a6sBR9Qj*cO)@cH(Vq9_=yZ!QsaZUwzK7({oQRdv*S4gO9lErddnY{Jz=mySMK@ zZR6*6S^JOa+YNhr!#m37Z1w(YcfI+~BMv)l(5VAXE11{rv^Ix+{`IYEm)$#Rk5dLd z{d?P|7N5NQ=~I^;`2N}p?mlqL9k0xrJ>u2T8^7E2fy&RGT6>GW&)hZY&kGm6xa&iU zcj~v^h{+dBy6wY5&i>Qpyut6MU9{!0A*U6LzVqZ6Cx7?!$>%li(*Bi^{YT&V zXTQDYp7rUxBM)eI_-ETLt9Wyx!gtObGvu6MYj3drMXgRg?ZKUYIDh`ohcDXdU-vJX zw&vwGeco->mp`A?YSeD`o;joAFXtcA@rT}fZt?QCLr<+-xZl6pyWz0`rMey(3;(caIl8!cy;Pjn$wBTOFCINg{U}+fq`oZ@LWFgitHL)8}_NV2tzNz zB%YG|e4)FzQ&CO{aFSp0#W649<0bR(tQ6~bpKHml!r&UN6&Sb=A?DgeoJdBATt7)H zndHKvf|48(#4$guQ1G2&`EJHUC~LTm7uJ$MS%*)3Ci5s%%o(PANM0@~@tR7-m8~9rcYe=zIvzN8kJgWW z{>byUEPL68mwJ$y>}2VKs5 zW%_;p{%Y>g=f3_@`*Uvj>(MtZYIDwkOJ;sG>em|wzWrgJ<5s__MQ+h?rN>>=dZWG9 zdZOj4XY^|I#A7df)NGB_ZZFy7)vNYQuU}Po-qY)MSZ&eh_RZFPqS=<0bU&_T zPP04LF5UFJ-P(S$-Da(n%A z^NEx9eE062YfZfT)B#sFUv2p18xCmQXZ@-zKihKQYfnFX&c`1uEGv7jN2_~Yz51&I z9{QkDzo{*!b{q7?@-^1$*ZI^#k9zH{W^J0UwOQ^Kty&%NYhK^k8y&e;&sL+?{%(Uw zKW%#O>Nl@($X06{_rTsatTS(ir$*eg!)o(-{xtB?Z8vINx=X9!8+~-{AB+1HKeyL9 zdHlz(qx%gXQa$s~_m`bqJbC8f?|#~))o&FKo_Kq~)e8>2=&bAe?$Krc ztv`Q#dFjNPzWo0E7oWSebC-+XD*pVa#4Gt@u0Q?O?+>Wh>&g+E&D`wdTOZhXou^+f zI%@s9F5P6}t2dRdx5bfn-?YK`uP@rP#m-mma@RHYE?Sy!AG}YgdZxbnf%)-D^Iy-s3%*cUW_I_sajy8TrbY zZ@zlMync_r_(Z?i9hNS={PKBSo?C11XU#6(^YgNUu1zdE?&ZTDxbKILdUqT8%jC-! z}wk2~R$lJ{OH zefIY?_v*gNk7-`OnE)ZS(xktvX+H_Zr1V zjXZ9~zj}5)^`$+RE?n|l-&s%Y@!rT1i{oTL&dCz61j;omZ=>zA#(f8eR zPPu$)`;$NY?w`#z{A9;*r!GCMc-|Tvk2&M1t1mp~^?wdsShYs!Cl7A>d(VX%m;7g| zX>;z|aQj^z>-zF8rS~3x;G$h`?^n|Q*bOiHxV-C#jm|mgj7<}39o&AcTNaM}b?64O zdtNbo|4EeYRdWhKK49$)kRf#09e;nDo>@4m3p*IRer@78u}^}43xZ&!cu zK--V{|2(DnVORY0+ZR=BOS(Kb>7)DZZN2QhPbb{}zc%~*x#*DTk3IA87rie1W35?h zU$aY4ewV!swyVhR#oqMbZcUi|1^A$GxT<-14-YdJOM+ShI(|EZ(X4S!*0{!HF$q&)&1(#B18U z*)y7w*Dce^&-iJ6i#x_nd1%qAJ@;(2&PRjaxpKp=cl@>UFW*0Y&60b@ zUw!zE-~2r4n`|9zhzy8?TKkjyY^}N?V zJ81edOSi6^aQ-f9UcR_{*|J}681VH$_uhEVf6ln|(Qih-^zzF)_xa%PxtqV-ZpJQ; zzT0QL;&1P)_^MsStfgyiKY4@6W#?S;*U)R$efRL2zg%ONUi((=wMFT+L(4yZWyUdU z=KV7Hrx{;=d+0x3y=Ty^1??t{zPa#(-S6%7-KhRA&u`!Rv)1qY{&+!or}Ej;&b)A) zEeeVz^*VE-15X{i-np0TaCmNB*ByF1veOO&&+a$4`^5b^bz6PFrZ@K3cedrjeKC#}DraP3Vl+5E`M zkNfgJopxIHp&>s$Fn7;MtsgEeJg{%UR?oEg__(DTFP-!GGly5ryJYCc7p=YZb?=6mABIcU;LCr-P4tBQFep4s$-6OQii+SYrP_sJW1^A4Bqe(Vz) zm3}_5blWaRF1YBbSu@}0^X-09ww|>9%r$#n+V_bE-zk|h_N&$NX79FDn;V9V+@$|JpubedHrZH1r`=aQzN9S*I*x(1cJaF@^xo6D!q5s2=ulN4Uoj&L?@%`TA zfqtmlMu+8;pjUOvGY3JPBo#BKs z58dr9aF+>VE33y=49}TTfE}=orR{S@S65G%w0j~k&OYl{S#=av^{c8JIe7&7TzA=o zk)v{kmyH<1tv3V0-OolH>F!B%vwN8P^gr}~0sHtiSj`?LeHq9`Ok#u24g(%<_g1-& z*P~+8sPd}v@gvH)2Z-&b1wEW|cKa{D2t5W617lvh>ea91LS8xQyo_aJg)0|Wo9 zn1b;n>{lixl}{-jpOZ*#kR--eU;*mzM0X$pVgw)*>)1+036B~x2RrnMvK(~vi4i#o z?6?wQP6C_0qjM4!+^5J%jLAui1?>nt01Lzuk~<7lKm{ro(~_GFmMLx;?o&}UshX^V zVb2~&A@=<`6o7=h0&E}V;Y3cOqO(zfZD2ip@va}t9pMploeqVicp_xIM-?jDKm=#+Luz28CpTk{`A`Jg=%i+LZM zf~eF6A2|Hz@)6jiH#;l^#ct@mtf1e2KF;s=-Ri?f?78EBlZJL2mb-qNNe}evwAIj| z2Mk^9qpMpk3-mDUWaZq&>^{{$t%aWcL23Q?^`Zp@`3E02=fR=-Ubbz^wVq$l^Rtl? zCYDZYw%XYnt@~#Xy`pX1NNzaU=AmhH^4ewN`ju|*bnbfZ{kYD|vW@^#7o ztW;g%bH&`NW7uRqSsTXTQfm73QHXN<> zXKik^*KqW%OxEUBh7Bi?`mr{x9#)pv=GHU~C)<_D+T0>4q{=JASqai`CRJXix>tGV z<=iba95OipE$b>z;$g;B9*UOa*;7rGhYjDz8V`G$roJo2>aI8@s;Tcv@X|o9oCJ>Q zd*`4wN$iixLdreB>}QI;#K3^M52ct}_@Uy&2@}c`U(v@yDpY-m$vKHB4OROwJ~v_Q z$6)BK{qjBaRJ1`r6;8#UZ{9et#c^uIUx?z{mi5>}P8RdUP(Y6y?n$FyhsgVQ3iiBQ z>4{>WHXznVc4;=WQ6T979<&s{IY z?zQ8k$s(7xO0eG|uRH`6;kKDzft7sSBEPg#$D(|mHb7|o@RmfiG&FD|_p}{f` z|6E1>E$?o%#;l9Ry?xIn#RpWbJ7wIMqu$+M`#m~6w$~ZM@C3>p)x^x7;ZGfBPrq{hL6e`G z`;SAey7_|1`|mV-%r(yq8-CZsmp=P?kDep$xpe6z#2W_RN;AwA}dOoNvCHe*1$n&pdIvF+HDoKbhM1iyu#_-2Cug#@%*hvoFX0 zzWxqZRJZ;2!wmdc8J(z0Hw3b%fT(|26%}mtKR`H*ELX!OuUm;c4$2KDw%V`{Hlky6p0cN1wT% zUF*9$G~4R2w^!@4q+8WvhZh{Z_ZAm+UVQIb9iBYt)6>Xg?H+W3yXpX@dJ)c^gmbqTb7!%y3Ow$Je=?>}eO z+=<6@xp>KmvmZJ3!?_=Rxn-{|#d{s`!GWC~-}AbkTeWWg{x9QRI=jV}NB=(gz(*FZ z*Kg3eH*Qk3&+*r6QvJwftM%M@;_dm{zWLT?Pn3?V{(j!&>kfHmuUAhRc6R%tyS=#C zf4Z%?#?aF`T=n|l-*j1WON&0|96MzAKl)sM>tQ==H0GqXQ%{@!%potmJN@)$Mjw)N z>_eL8&TRSI@>@r}d0d;_rXC3uw>k0C({G(}eb4PazVD8o);jW!gBAsua+&3-uLy+v#vb1^_hFGF+2a#^RE2z%+?=uJ+1TRYv%9Ve%XH9ikm6% z$r~4cmb?8%pI?2+l%J|wO+Mqri(0j~W&cx;-)7Nj)89S%=XJWA*t^>XyS4AXM(N>8 z?|$UESD!uL`2E)Xt@k@ip1Cry=`#l{Iw61GO;5Xa;8VkwcKH5+?GJqF&>4Z^ahpMH zkF6|X@wlyMM6X=XRJ}67l~a>Nw?+XcBGPK3qloB_YEDG3eajOOc@q@7fr$xaI5rtu zJ_=R046Ed&&_eVO6UI))+n4osfFfDRn{HI(QWk3FZ5BbB0@QvBwDR~_S^x1t(1r}M_g z<*e7^x65|eV(`hUZE)AUKWtrmcEJaaZ#J;syta4FnYp}W<;dwbY`XBJCpr%)*nRR2 z7jJ&;H7C5%?U7CgO=&yts=VAU$6fvF8N>fGYTzM<{Pg1fpD&v>_w*ZAKVs1-Q+J+y zNomJ759xN)Q`>Jntoi6)yMM9uH@S;%oWJCdS1Vf`vHR`K4!P#E4VKK@uR16U76!C7 z$e;Bo3A~i%78d4YNlt4HUTu>HhfWx^!6j>#eX+wO$+M;0*VY*yNNGShUeA=~p%r&Y zT~T!wRJE>{It!|2>zUFbkII~LOF6dOutnRd2j&>lx_8+Bv-cHHQFVLYC?z#?cc^s7 z&@q5?hcrV=2uP>&&|T6YA)u6WO1E?)Fo=Liry%f+@jQ>u{k-?R^{(&z)_YlNX3ZJS zoU_k9d!K*(e!q?U`IkfM*Y4!Ma}xajGPHi>H2#}!*uUv<@GJlA-`vuF$I$xOkNVBc z{P#BV|CgclCx`97#peE-EDfv-^55+pz$iFB8=8OWsRhGd|9{cUpY8UqBMA;(7*HNY zBK>=+ZvTF46|D5}Q$PO0k;J#;@IMtv{LL_d2bPZiN)r1oF;+i?2tNQ;uwTO3qYf(! z{G#0UGt7$3`oBb3{fG}>uJC^Y*a{Y5{kzLLKSI5~tF{C4_WrWq#`6n@<)6&?!g5cr zdkE{e|Dlzg|D73M7*_A6$^OF%!OCz{EMMP#d;N=%-#7U4Z}gT7yI65R5EZ)AnXnYw zzO1~I`S!R?{WAqvmJr@iOks(8v{lGg^*VlEsi6Iw4nB+-YdD*E0W zRhH=y7hv(*Q3q4!h)TS|$>~k^{PZo}cL|~w6*|Q^o=td#$sen915hq2iI(+^+!;b%Yf*H)SnR)XN-=i&W(tn|Mh z8^iM__wgUM(%-=q|3tp;Z(8Yp1P%S22u}4gbml9KTnfh0pkh;o*$FBbDUwa|hs|a) z`4co6c89(Lqrc%}{yHl9zt>*>0G0f|O5}YI{_e{$7-9~DwsvLJC~CP{N^ zu$IeL(UD(dfMH49Z_dZx$N>L?UJoqY3gg*-E!1)F!=N)9955WzS4;9|GCWMKXkVh`)9z|8x-)))hYoJW|7h813wL;70}3*(CSauXMkB_ zi$C`(+s6BI#PM6q&BF?-pMHJ&?e#C&_`e!yPI9ULBB#kVgKF{1mi{U9_~DK4u)@l7 zU*CRv{Yx$7_=`}}E>;)-%R>jBWd)!KN!$|Geo($@qR@O(qG>CpLvnE>VmN4V-;$v5 z_H6VADofK5>Vo@juBd@As#RUAlDxJ)ryhrDjO9tM(KJKJeFHQcZwl&~~+whtV^K3HGVF`-S{LNJz7!2V@l{KEDAcEkSX z-L+r&kpEln+OPb`|Cw{`SDxPg*1PtzxA#B$3Hc|t?|=6b66Rg}5v%-}y$6%v`VGzf z8+VTlCLiGhW7GW${Jq~iR#2JY6;|81)UN4g>?9QnBe$@xv~{<*}Lh6DL!c0>-cX;NK~Nzvr%gE^V_pQnA4@ zW=?+~#o64b*xbLCw%I&?S>67_m4pG<|LENxTx^&cAq-aiI~)7^CJbf$UG@pgX7and z(^pxiU&=ZCF6H!9!RgyAf$2E8{e^N947vYnX^*c0Yb;!_8a*G3uLz@oep7n;>NmnL z#$T12z7?{6xRT!~h*GNZa#~7E3W|#MkL~$c)XXhiY{9TT`6ZY8?@;-Gr8j@}7GWsi zKYHnxK<6LGVIT|+@sr0lZ>Onf4`s({+0b^!Oqz%9R_fCZj?D;?A*+3fs=JePuP{Ob z2XAHcX`3$>?)(jIvSi$l;Y9`@qJRaq4rHGt@FYkDP3vcZw_{gsA=~ztD7TsnrH~tH5xKpFP+5 zY*Chk z*htzS|8}b82F)drEucZX3WdQ~4#5Ng2OueeMPiKb`g9hVGiEIf{uw5n@pBEG`^Jg% zx{l$!Z<;0Kx+75;PmZIPeNxJjh(21O9F7mk9T!956K zQh09JRk>Me8m--qVv3gW*?yBB?=wv zqQjBA_ly{Q5VeQs9uskAbD4?4Nk0+p8o1xOhlpz9jmhz_M+MQB4CH(W6Krdx`44F3T&F1D&UGCpu5Vb)HOTn~rQSOK1|H1$8M!1vAHt$;Y21Wf)JK?U&2NH$ z57~6sblHBk9A6~?zmev@+t_dNfUYoe@_z{gTC>^w5DENBuK&N30)IEm-?Re1vGKnW z^MAKFKWhj6VP)A|zY7Vnd46*ge$p2CCLr=RtrASO3uYaDWtF~~(mz=x8F57|UDY2} ziG%&WFiT(4V&BV6e~~-+LrCy1&C+in*?%U$KRfw|0Wp> zD|G%bOaE~>4DKIw$G?&K`{%)*e|X~DFxcq7;)MT+iU&IuOgrPxWJ(|`H?8%p8;JxKB>XA?`1Oe|xO{ zF8=XNc;x%Je)#P?Jg{N=ljEMTW58easU+FJ?CQ=lLX0Di!~%}KPOpg<)RS-n=UiBC z6OR}`iY4MWIf_M-;q~{u!kSDvAJ4(Tmq@P!-xpcSm*Ec`KWMlfh+t}_!!f3-yj}7d zo`JZW4rk?V92{+!@MsaaO#itYwCSGbTIs`4I@s33S79_$RC2v;ir%YH+Jd-W_hbu0 zaF?ZPm(1(3Venjo=e}L;*&2-LCl{$k9}g(d_fAeY&wT5fI^E?au-tB$)`{c3WU1^8 zcbw!dbpQ~7D{C#8l$`yNGmKF$0D zw-xBLpqGY6jVzC0mr|@Id7`!%?xUGXbluW~^2zb_J*~Y$bgznp`e7R`UFcDx$Js#| z@Y@B+)7%od4^E$nsqNYz6 z^Z5P9Jx;A7tUw0*Y&KQo1UF;=hTjyKu~1iPcxT&)Xz~L}c7T?Rv{0IaBltc=5CQ*{ zm|V{W;{#6u18+pfb~VBDExhy@n-9Pphk-6+wB4X?Od@2B)JnVv%va%N1*^h5qF$EV zg>rMaso~wp$ryaz*EaQ$&@#M;H+{Z}Qt2wG4_i#0R>ZZsN$8b@$vgPDyy;P6&IqWo zy*@Gtv%xVx7f+0~!68fwe)}|ew&lR-98Q~O&&2ak={@|zvao96{^9JlMF5>#dU>lE zIhg8Tvag}a@AcT7o4deNpwKMr#*0DI!L5jd-A~|7k0j*Oq&s|tx@071it1(F5Y8Dv+Ms z6&aZE^x~)@L6TpXf3oZ2sSIied6>vMfyf!iE>xt~~?;>h}GE>|z+f}`Bjvg@n- zqG!hyN-yDILWz&F1>A4k>>dl^XiIp0`oJ+M%Kw7p?$|55&)O|k3tP7MwdN=G7N}Zi zbI+QG2QC`}wl+`|wc_cOT7j2ug+^iLYH=kvCA{Jhwo}gY3B(wAs_>v(CUIjKA1<1_ zjcvC4MGAhHvmzyMDmdIPqL&7FXE+f3xtYGLydDUFL~_d^uB*qtCp%oKO!kwWth2$X zd~W55vxr-qSIYE0nf_<=V##`ije{Ub^Uo*C98MW)VfM;R!tYvv2>U9`Z6rKGJz6;3 zQ<(amqg9^Rd|vyg4IH)QHRXD$EZNzW$i1$MGhH{XTyVW*gMsVp;U+IWPh8o#;Ux7M zSR)BNBymHVGB?~4y*U$&J{v8`$^u?jVY$mdZ!Q*l_02A^>J!c(O-ftT1dim`Yy^Th zUav zR7c4mR3BB+>rAoo9>oSRYNKj%H0lf6nDOr*=tSP=3(*}&2y0_3Woc-6N*+EGG(y6UdS$es`K4RfHvxOI@JPeI zum9CUbfr&{TQ1_3UIp$;-h;EeuhMp1O%Al6EI4<^um)lz&J$z^8`jOV zvlW*1MeTGbim<&`ji{cN)uB+5Y1TgmsNFIAn4D7f{4L$VxFvRvTf?#psx-w!gF{9< zeYr17aR^85kl8Uq7!=x@^n!gZB_u$dg}4j%euYU4e=A?+u!^rJllI-rc$pd0{kfd% zk<6PC8G-1~`bW@MLSB4g1EKo|0%{X?^8C^1xjY@I!wwH`zpg5lDJMXq79wbCVL@CI z3F|n2wbA@Oe|Y$+C;kp0oh}i7;pZ~%klRCuN-jDam5vgBA+O0l)w6s)_ts_hRC=iKiH&jEmlqNXr=;)H;l}sWB5?E6FpgU*lmP@Jl zLw?+gaLG~uNPZ?L=CNj7djae@R2tf;e9vjDH;|!cvVl%@u~~dZ{7s9l0#l-?qfwm9 z=t|4cvni>%VSR$()Vr=mVm3q;zxe*$*-d-uc(6cr45_7IErsE@n)Ub_ z9f`y^yP_y@C0F?m!8 zr;O)Vqxwl6nc08gZx8Fz!g=>u+#?`}#o$ZECK;Oiq6{8kw)`>}=)X=t=0r1Q*3q{f zLHV*`u(%+CB+TpDT!&O#X*LhpG_R{}YvenP7{{_YFIv)d-Jc&}5K z5MEr~7wNckqOB}iP++IIbS}mqyF0d#II%$s+0t+Jbv%BE@EK0-K30zF<(K3ks8dn& zgbwtzt=1w@7-`j^j`!;|$FiPs<~cnarx$sOFCS9e(#5%YQrLK`xGq8`_fgy#ZZv7| zIc8XZCl(PY8GU(np)!FPzTbXTqmJhcN9nO&Q`##?L5`A5HLiSK^NG#wxsMiR0>8_I z93@Uah~H?F=Ir`&ERn5cYycx?M{2GKgEE2B@xySmDC~sP+8_>mZy6ypt}kXb8f>!Y zt%$|!nC!hg*z^ib@8wKX-BH72)zN+FC@z_AwLEir9w{Ltg_97F+O@pccqvD7OGf64 z?(*Ted^8ahEBuou^=bnJrMV(QjJ&f(J<86Jo4_~c=`A$}cswS&YpgF>G6f!PXBL}3 zvtdUzkGoCL?j9p_8!_O1m>$cb;Z%e$NfzB>n%R7W0X@!ksUvO z!HkGTVUy+FfMfKqq_}xH03TLZD|cORXLVeX@hKs|eGJO&Ve*Z`lM^?WdEG7}9ZyuXcDEy`3 zMcTcO6ahl>$*~`LlIP#2oW4d_SRfQ?PsP4n%XXe=q4=Dx_RuXgpHcrY1WiA?kVs#eX=azuAgBZY6?dm;`_PQ5<<9Xt>I1< zGEW@D6T}_WBGj!>r48RjNjNcuTDRAlasF?Mu)7dh!sIu~A0fSnv}QP@dk~?*$HXTT zfy_$*!PX<_uc_7-4q1qEM8Llcxl{DelghiAy2p**+u(lR>VzD0ruG%Ws1}CSKnRKV ziJY6Cmhr)51}>q;zwVHsUZO392F zS1q#qGZWZFYeCV4&pQ#!H(XSZ8k$i&&$zQ1i2S+?PeE9DNy|A8-SsTT?>DMfzFoXa zS83fN9B(R#j)kQ;sF~{=%c?%_t?OCJWZ%kteNcv}?EgtUntX%&alrp8*n;Vf+A~1X3*ciW(^AeLTVjqGkAr9_9=D0-=$fR>^ zFX+&fSd3qT3AmKx=!L@+MU(GqIb~MTwe|C~@_WM@Wz_lre8eXP=)Q>6Gzn$D+>wfp+Ds$do9x70ZSFl^3AtuFXqc>xSL->k4r4j9Y-9|6w==-0OVZtx{V# z^OOkcHBdq-pwCgUtd>nGv}v*ZAzEW!*}KC#AfkncCHT+cx8RHrNccD?vI!>q-7w*v zjIt&5LF<#R-ed`m2oV6E;A{*-L1{TV5Xp$*!_+BMJ=3qi(rXA-njMBSn1^Qbt(X4a(5IC3&pzDj>EAAUxAW`d=sI+q+dLWh@+xt zG*=5QtblBTXhiD4)$t~QG6C|)yM3Ji+c&@}EL^XNqCO@#>al@6)&Tr$&4lI0Pr|EF zH}U0{Z?`0Z&}~UQu(0RTn!svwZuRJ^mE5;C^^SLpp&DW+O}+Ms(g>kY4HNwSEQ|d* zV_rx*5z?A;oLO0iR4vEzlH>%wl@E-_?A~t2<9Ke!?2fGr43%62UJPn`os?xZYG?dG znZOD&-r9uwSGAuQ0&I}DYGcBs)dp5ax=pz^AczLIPVmdEV~@H+@?-3Sd!h04dGB`M zG6SX!kcB@@ZWx~^Ug7w6?IDbGfyXk{*1+LBVF4&o(0Eo9V=ZpQgy3EYlC&tQuwf%K zm(14@nE|`fy7j)zdeavvDk0>n>=s4<>ifb?FmiUg4 zG?~{*QimTiYE8@r>vhR$+`3vev65w|3cEitYJ0I3<`+M^wr4>&I>OriQji@^mjQ9v zNjAV{!*#VO(pumd9c$uYeP=jEk^ZCNRrQyCx?B$!O%*rph7g!=UPrm`Y4=1g=-umA zUn9h`_G@@NdZ*+eor8O<5y{fasf=l^DQ7OxZd?tV%UYb}k~N`<^cRgy#ER%5nKL|? zbF%_Ai_x21cf|VcoXLa}h2OQk5tTTf4`+Do2uuqboVH#mSNfp;)_* z_He~2V+D5kT3v~_`n??+A(ov`*=$r2cPo3v|IlD2(#S0>+SE2;D_5s^;EuXv8L0PK z0cj8`X@!75fjLI%3(LI(i$^|*cxn8AZkP+hAlBE=H@IM>Gb$T|QzEuXa<d&ZG_YD^ZXy+{PJ ztTHBOVi<(&-c4D)TvH4+JkRR(r81Gc?+BAFx_gMmbU_kcDVq`5?Pxb7&)TrDhx7hn ztH)*=DV$G2pjegYzztP=V!FTXt@k+Tr>}Y)iHl>o()!Dh)`p zbw8BmQQEbNv$lBkeyHL(7A4CxjhVA8=_vV7!O2ggREzvfq&S8~P^X5iI8Tza&6Uci zwPo9_n-^)#(Vd0~C*K&1T(W+7k?sFF*!A5jmj^aRyc7J?Av(g_3FR`@gTh5G?mFUb zC9B%M!kCUU@h(i58e|X+EoR`1)5Xz#1ED9;1yO3_r@zY*w5LQhx0XfFf)eH<+ZCao zgk}m3ppoc{7Hp1t)QErV`L%8KdGiNeIuvg6*X&XNFG#_aT z&J-qQKT^7%JtEk<=Nsy3v$Dg1Xu~UKvf`FDMZ2JV3k4%n7Y8}AsXrf#SGnUlp4or_ zH-Wtl*&LUrxu}hLG{;Pi+j?PUzrVRh$gdQeC0S-_QiVxspkIROa$K;QKFv|dEQ8Sc zQf;H;@l996)y=zF?n|hRM_CwL?T9=+HHwvigVRB`EgiNP6WS)7orDsd#0p#3Ik!Rd+b?pu3xiXph#*YCOHv)p1j|a z<=>L%{C{7{{I{0>MampjB>Bfu=3kXPeovWyhnN4M3;4HF=CEJ>NSX6;!`L)`N|`Th zr{x2yg2AaH&$cpomhu~J(wEj>`-iH?ppuK>lz0rfVd@%L#0s&#^At@rD+kq{pV!F&hD&{0>jgKh#Ea`HY<|(Tq0%x=o1T<-p88)uy1+i-d+IjAcBmNb z{JO&}NKPm#%cs#35$@w0*tt~@J6#Okii#XZmJr>hvDVmXZhqwuIh>$1dd zEWnuTvwIeGFAYYpm{sXWL3QOusMu??;V06r1MZt`SKgniK?(0!9g8ty?~oId?`{+F zPxd?rlIi!aFcCVg+d9M>L-$F|+)(iki(P&M&~ET;+X(I1d+p(IqP2IXI@OAv(7MCQ22}Mu{s*WIegg3Ju+f* zm49s;vT)-l6{a(9-$6Z4Rsd?n0>rh<_v^$VgwVIiLF9A*;@Ea4=64{Q>PE00e#b+O zwThKwZn9bC1rJJ0=eojlSjEbmUr5o+9W$u~iFtd@0W!fw4xyZvcWkyRKAnIZyq~0( zy#i=^pCh$!MneW4p|4`OTmwJ#QhzWvbKPY2Vgla@utI0A0{M+$t7@u9RvwBIaXIxpCRcZ>_h&4k<%eu=F7|6vK8EiWJy>De zyL)gayQ4heptl9<$XZwAIoIm~&RYhC!&pmH0j2m|Jpl0o6l@(9a)K}Sx>BW%pX(|P z={bZBe4Vi@qOa0DFng+M<6mW2l3#WjCCp*cgWv89m)3SYXdey?> zat{TN`=k{;EgsOB&gjThdJ{3rL>7--yU7OcQX1At7ETf`jdfR^TiF$Y5;IL%Oyl3# zo04p1l1i(pGdU0LUq<+-Gp;&LIVS?iFZKsDmvg$Hgu+JP8ck{1DiRi6(T);8w9e15 zgGAuV#`04{*M#C=kYnlj9{dRdPpmoc%h~eNr0Yv4*2;wKO+j|5IK-=}2E{Gjm*51UtF z`HlUjoiR(Dt1JV~eTsCrAZ<-j0bsKw5~L*WIBPDA=i;5(%jWIw%5UtF5G9^Sw4C9@V2A;+A?dQY)_oqn~R zGl(jH&h^EAQ_#C5_WY!GFAso{s7}?fIvmcA*D`&5vOQz(XJoXzTu$h8ea>rz6*&)- zQklq8m zbbhQwQ5F*F_@s))n3Jsa!}^{Ns|W!yjoiFfEF|n;o-$^@qc&4u{Un1d!`R1xnQ2`54AEwBWMX*PDS>;TnqAA-(fp$O7%NwcI+D zcGM{Mi_x&ckCKqMTM0!2VZ;5Z*?vItH2egR-JmIzGd4t;{`QnVGs|R^7q_=8B-d!} z_-NlA*=Rx7ZqJpWq_(?Fe=hmO42KmIiu1Xio>yJ~H&Fi==>SDR4M81$V%zLyj8V z07x)?uTf@1SpI|u*epv7$gHab#|l1P_c;`AvkvK8u<$2(mJ{`?y{z)oEhVN}lD%1H zL(rQERYZDPu693(DH?DVwMSkoT+4JIzr>j;#g)){I7;>#wQks6WeTIbiotuqq5Va_F&2 zy}Y$ETlYB#baL{k9WKm*bw!19C?J{9&r0~com)N|bFCm1x0rwkFBS6g3bZ) zE#9qz{gLcXaTHRRgu;qm8(VnKcBc;>|;6G=O-c|m03 z;7=xya(Wz3PChzsa=j4jhHD`d&O}TqXcmNz00CKltg}0gH14+8Tpk8d;&fM)oU%AMn zQ@~NYaQzMdt?q$k5s>Tyr#Xs`W~&vSx7$lhys34J(04kfGbTtc)B0R7Wb)P3HltP@ z@XHCU(esb^DdA)E{D3AXkfmswb%1EZdlDo_=~Fa7udPt<^&Sbz%UlO&K2@9JCv2<- zHnBQ$*)A~)5jUZn#uPCL1>Rbx(+_txo^3AD0G3=HP$L&nMVpc>^5yG!(q=+kniQvl zyLY?E8V930gk2Who_%U&yQWeP-w7t2S}o1#yb1)H zDB9albA(|@c=7myI=v_x%a^Evi5A~TXod8aqUjgw`wZhaYcy#uFsy#mJaJNWwSU|~ z&UkakB24G{o++ra^oYE1ZE9yQn&y4Q5)5Wd1`y;+>U8pUN64+v9+(=aA!KX>nU8(O zQa-;$pUJwf8{9*Od%#dWK;8+Sp`l&vHZ>98wV|*o`3wiXIcAt+$!`v&UuYq7{)p%g zKKtM-BPuxY{z#7{TBH}QSh;Kx6xGN+rmB5{GsR7s&4_>rzX^*Oz2s63HTAZ zC_eWUw}9#6C)&+SxVGQb?2MUXgKsZGl_!=495;PD=3YColRaOWcYF-Y>+H@BbMY-2sJgmX;o_#(g`89Q6=Ae1FjO%Eh1K6iJ&I_Q!q z>kZxXc}HDiXJzjIR5+;IH;Goo=~%5~-el)#)@{QMeMNuo;3Jr0hWg<6I&g<-t76VE z|6`sd#HVmWjFfx*Ot9I$FQpQ9;)^UKOfH<`lS^9a*vG+Yeo0!M8u4`;;=@8%#SRb^ zXRz~Fn>^}l*};u|QjeILZ>plt$n_B<#3-t6?tXX;X++x#s3(Of-CJ;=eTmAVgS%$* z)AwLb$e^4rh)`6A8U8gV#CSHJn7kE=-m=GMxhp4N>sj14ns`UFa?aVvcu8vkgUr?D z9)v8kmz|x>CIaACdS}^YcR~_AkLU%R#DuiBx;>q(S9{;2-h$8uq!DzZbWzIG2%0RY ztHG>CA+I`!7%n|j#|*orx{GE&;zV?DvO|ks-BC0%RfUh#A)Yo2F01IANvj_dA6QUe*RcVgiWK@}&M7Dd@SZ zMQ!t1?4swljtiT^KtJ$%K+NfpCITd@F@2xjN9|a5NC0r+_r2-laL-f4J=&K9n7{iB zy=ULHa5<~`yuS0@gN-ETz8N#V(faP9b#q{Zz#AW3EyFgQIg?bfs>#EKZ?jw+VjihY!4PQR{AIGcs%3kKh!itty=Exs;SSw59iERR!ad z%B@s$NG936Y4H})Wbn$`w8w%yUth1?X{P2UlFac?RKio8&heLXMdX{%`IdZKIsQ+O z${rLK`eX7sxPu|iRdhj~l-KdZ?gu@cffQs}hP-4cU-q@9cN^~+hcRVjH{CW#>GYP6 za@gI7&7!P&5J)J-K39vwcc!vw5GP}E7{TmKQ|0g;5P9ASN zL4sB37~##D@=a>uEnZomLt-f)+4I*OUO){)kT=_|m-l97aPGEGEYHgYOZ(Xj1@Nhe z#Op;jpoVvx6cd>%L*!N=0FgZ`YIxENTOmtVazdCG6{0|@Q}K3ku*%p$s0~=uBw*%t zJ{X*Rx3?lx_99#@hCBMihciybG0Bsr>_+GlSFv@guIce?BI9KWS+<2_x-I{ z-Q(#A&O}twrgE#SgYtN#$9EFbiyyg>U?yqSZ6~k!)`F_;z<+o&{6R%f6n&Ely}#)m zN`S-c{Rmjri^_unsnKuI0&teQ8^v<|XwDDR-0c3uR3P%o>5}Ln4ERwNfIuV?J1RC5t;nb#dIMe7hkWG2fnW}N{} zuAozGJP~FuPkumo?d+*hYX*4b#@M)Rlw(@YwBP-E{2S8uu08Z ztJ;Ue;dL*U>l}S&>1jK4s6OyNAK>=+LS41BdNy?bC?gVK!_g}9egSAl-9+B`xxNM@ zR%h~xOd6TtGm&ON6r^(24S5!z>}kR`BE+yfxhkRYdBOo^iwb6!8I-r*t&2IgU`{8q z`M1sHy)5^2p8j&ZT*(Avj#6ko^QYgdCd|H?>PBs1Ed@0kJbL?Sx&ho)F-|WBlFAbt zlCkx^;A-?bbF(t6n_MMUUU6JPPOcCbQ1YXQy~m z%@jmLL{}uM+74kqwTuScIGKN-b@%>^W$9Mj*4T2s2K^XgpC*cEkU>p0ka3l=?5?*| z5mcQhwI_&NneDWGMACv;5zyfAK}}-5X78w^i1$%b`2+FifD6zmyE@dyJ&i+II6y|xE|exeOHF0L-#k@JbiF~?sbjuMspvcMOxZd7kX)eQ>I8Wy zFBMxLYszc!wB)*Y)Ku9v{RtY?J!F}SeHF8pfCVw3YmKL7@7WiG7KjCG*Yy5u25^`;bF@Rq)#<#TG$UBv_fPqIT=*S@H%hND}~I4+j}eer^oA4?9RM z=dg&mRR~Czx$cm=aPV?9*6<+{*BP#=1yI2l-JXV*Y++Sr1GdM_g{NpJRaYO#DP@~o zfxfwM@$tiuy>~=tNDl@-=eC(dck&$09&@m~WEfx97J6o34s$xC&Bf637z2Ss<1Phl z7BAvDsl4rYWPR#EXiL=%KF#57O_KPyW9Jw~ukaxFtcg!F$ zG)2i<>{bc_k=O0GSr5SWWQS786@xgCYx8P-N;XDTQKKUp@q>p1Pn|31{Bl4(sO99C z{11;ytgM_*4YED=U!w9K7wro`2K|Wv=H@sUJs%wD1%pNpDz3l8?qq#Bys!jXuwGU# z5=0ycm~n(+LmJ8{4|%Z4y`7!sTUkY~apU2kRX3~r`cMGvo;4gHtZ zG6X3KwI!Q})uVd6;xwMo&yahsCv3GXMOB8}D?-W)a(AliN`@qdPesG@>AjkoMv80K_G%E)6x_SRzU9uS+ajL0lFY}nx!I&oy{I3Oz45`Kzx$u$4L%w&iC){O6a+rEEq4*SLw#@PJM)iePE z`OIK#fVCCYhOZD6J}@WH?AsN=i{D+_`TrjEP z-=Qnt6w6_R``<^y--51uJJ)Y)PcHsHi`DLAVx(u(;!7tz|8!5W$fY@bXJ$UZmb13< z^5&@#sXNpNa698Jbkc4JS z-k6S3MceEXafiO;aYxq;aAryB+9yg3aA3*p*(WmVVIvko8pguHpa`7b`T`#$2t~Rh zgv2whN+D>zPc#{|=6KI%m7~w^xc>mimtl#qi$46K67e1-R&a05W9}Dnb7(Q*x-u=W zrbkG8n5TIA4DD{Z(_hOIV zmqz8j9Nu^?W6oM|uUKyLii>98x$*#deH#OM%$uNGIa$pV$B_4DSBlGqfoHw&`p%vc z7GCN}y{;v!^fAwTrP?E(NW7XF;Ax)N;(nP?#C>~*gD4d}E&}bIW18>D$;X4W`X^5i zLV6!a%i{38ak%#&rS%EA>;CbMw@3`hM8$Pl4twU(o!-_as)U#EGun3MGPIL7=8;*3 zuY%1$=rNd}4M17{&bnNyuxMKXleVeyRVu#C&R+jALFXEXy`JM{r!eWT1j*xUf1h4k zDL*B!Cp&PrV-4h(kT2kZ@h($o!*PqQ4F zCFp1pnUKhx1T0tCDQ_=#cZw9HltJ^-uKn|@`YrbByO131t~&dpj?T```qvLk+!vW! z@$!PD>n~4tm*@_+r`E0~%Ri?1^`3REz-emn&oG-<`L+p*DpNf5zPYX|eWRQ$dIL{E z8T;l|N!gQkt-e<~2lJ?HPeqs7yv{F9-imh?75R{KuJIC+VbzX!`}R0)N1b0xyKD@a z4}`b*UL3ER7j^8{^GA|8NenZ9g*l?sd+O8akKeO9BzUEQln#ol%OGKww=6m}=7Dw@TwSWnCNtT?HIMuOi}cj68h3h1!-ZG6H(35cBzi zc_^gejNXlGBvec;ZF11cGQryxyl&^(I9g-Jnzf24*UBiw-$FKz&}lEKASI?BqQ`Y2 z>|V>~Z(0W`=3q6xkRvQr$U=oQlbCN~3{@<6O`LKMAF~QmBm$RkKHRz+IxeM?PpmPo zOPDN&P$B6(V~uUOZig$aqwOGKT*T)4l!$p0e;7D3ZXP}f zT?J33bez$Jk1f&)a*J#*f%LYGsDl9dr%b&a}L{rZbz#`qwx)PG@!Z>m63HU{*8%^k{CHkS-^y@(^#ZU#={CGwl#~|8SxGfIX-Vx&1#AT z_u9Bhx^|o>ew`ze7b93i?CEyqg46TPj@?S!;d*XOmi$!JP@Ptv(--;vc6bDCNxgX% zF%!|Bp2eU{k2)S#_Ll>Sg;E?*NqX_j8wIu!D0Z+CPi9D%$CloC<_Hy*#AuHpKE730 zp7?o`Ds7xZFVOd5;zi1Wr{>cZ+jZ-Z{El{v&O96fw0wC*teEkzg95n`tMdUKR!vhs ziKBYIuG+ErnP{d2tU6IOOcA~rmDXLq4lBYz`A?Bok7%B6s(xhtC zA6C#ng{9l{IGzj1m3+bJ^&En3;h6NB&JOx*I}+Q*6k$cq)+?NVTjVVm;WbBzA0Za? zHjeWVCEfM!#Zb{*%jxyeBt8hbc-g~Kilrg06BgXN<`|~=e|US#u(+CVT@#1ku8k8U z5L_B}OK_JEq;YQu7Tmpo;1=8o1PHFd8YcvIY200cWcvH|K4;E8bLN^e^JD&W_qDpJ zmeg8xt-9a$c}hR}1_}41`HNN4;CU7`O~N;|BfDT#6Xd@f;@SnB(Q6bb3el-*1_}Kk zN|7uX)gJion_-`Q8?+I~mF+IB*CURa@SX4E1PtqF@Wq@{EGZLyi_lORzsa&P9{bx+ zv-i0>Yoe*AE1yGI;Pi=Wx}lxL`{IclD_h#Xc-HH|wGEns#ueT6onl7*O|XqHqnO?U zPeZ@7CYfNU{XU_WmUoATcpCS60I5I+iH@vixS~^PcXM|c(OSrgF3J|l zF{z;mwAS{@hKj>S_)kdCaiH{6;FcW($~CVawXh$rZMUDCPxne-ZR5gH+Ah%7M6E0+ zDrrG5^+5vl-50fu6PomZhu|x{%#Y8_#Bqc=?-*k5n21&Gw=m=?YMmrZ6A=UAkIomg z9Z$_qyPo5u-7t+*=KRgw#~axY>b!Hwyf}km-3YN*`Z(CW|ClkD3gN|qO;`<_Vb^OZ zfZkI$mGJkTcxTLgZWr^D@t)EWFHZ7a1XV>Bz5a1dCh8={{f6^&7}iNRNrUB*p8YoI z1KXSPdMH8tdM?x}S)b|G!lMcl8%#eCMnvK9o@a^+>s|fms(h%2g8Tw8GmZder*ClO z;uZP9pQ(W&hNbS3A2|DVsdvF9(&6Q39!33y0SV(=w?hXhvfF73*0l#0_9^DC`8`hs z9zz&lE1nVgtjknen!6|KA2dKV$CH(G&3O#012pkoK#q4GVZ!TeoGM4t?P`?8RzFj`LMsrBcZ5^>AIoK}LHo#8YjXxQSa8)U)<-fvO%mj*JD+C~X9b60e zvaOHSzV889s#Z(%W(}XJat6=uA&Z-JzlB|O{upDmVI|;6e@;k$yGN;%^w^Bzw|$#I z>T`)n6Uy&MG|z5N7LXcQ{JlrqI;q9I$%jJ>`Hh&bXo>*&iexrN8g1hvdnt9BnvYrl zJ3pG;;+wiZ9qt^@Z;QXWg!j|Y`@UwzWBK9B{*^k`4m&A9$|7F(3&C1zO3(Q{o#ho! zDIhqVb_(4~-XbBCt&XV~`p(7`pU?q20TKDg2xdvs4E#l|qhb+VCd+w-5-BahYXDGI z@ZCKBX#fIS#P4}oGo_OQAhv-G-eQRwX5M}Vg;W*?V57WbL+pfv(aM}&`#hqg8PEw& z_a}moL-T8I&$r*8hYR=*K*u>?5_CBn#~xl#zN1kh!Hc|eWSO;jJoN0rTuQsz);G1i zZRp}%d(G<$18SBV7i1&)rA<4Vz0kR%^bm$=r(KBzR3@~G=fM~C%E&ZQu~WPCG$Z3) z(3S01SV$=T-gkACTIelA>-$&22?1cD1o`_v^cxq!i`CrVphdGQ#!~aPq+6P-UhO); zqVTIzo+@@E8rrN*%A*A7?8)>AZ!zZE?_!=3&bE%e9IlhFq2oIP>hQJQ&yHAo*0_Uv zPSTw;p~%THunj4!hUbzc5Tj%o>KuJFGNEs95#bd1*_{FK-DfJTK2JB~MJIHKDrLBA z%OI+_plAKu&BMX4S730_msL<;AJv8Q5x0U?95aOyVF@v!U>Sm(#Oe`RABb-F3>?DF zrDlh59lWgL1||VwotU!|6Iq*W_L&JxGt@x3AJh z#JsUXIBW)azJY5*3AH|aA_>JOpoD(T>1douh`uEcQ=+dMn)%QR!dInCh^9|{oXYlw z*9b%TSa2a#b8E1A5;PcO?ZTJi(?U^mDn>!4=4CSrng?zLbS zd=^|>C&MTl42%{NJ<$0;SzEo2Ga*l&N<~wa;GoFQzX+jsLSdRypRyY)(_@xbt zry2c3zfo7`rS7+45I#spY4!c|;t#C+hu1q-r}*L3XcG6bo8TbE-$^vJl91g>r&=#k zQdq+vr7ms|ZOQ3V_ghNQbXTlJ?h88$_@v%;6qJ(CsU>z}$UNH26zN1<>Qqy3iW;qH|yJ`sfu80yDCp}i*k_X|i zn!5yAh@U#ii1vaKsmAq9mc!$Eb;!RUId47Ae{zfuRpmwSi9Y+-KqnqC5)?`D+)qlA zK+nq>Zhfkkkb6x|6Z+fs(0OX%q#_6!6#T8CKkSHy(6XmzAIu_hLpMYTNYJgyVfl!l z^kW%7b%p8GM685OZXgx|bA+D1t~dvnaa%jYSlX%?7^n8K?Xij}^> zR}N3&N|xP(*TB+-8H-k@=T4TrT-BPw%MrNu)RoqaA3+13udfGSJaTB&pTIPd9bGS1 zj|q}5FMC*GXe5PVg2|v!7<6y`%vAE}{*hKas}25u$$l`pxP5ts(n>sC-i&L&@zSBI z5$xe?$L+K4BWy2^c72UAt(lu#Cn8%nZbx4`PvY;N%in=7*TZ&!n>^*1eHH~)RG}4I z$a-4UP4)d)2*GKujE(;oTJo{!Wn(E~2BlBHc&0khz|xJsXna4%(DRm)cw?U;9=L|V z8PyD`hY1T$vV|?S{lL6DSXQ50C||nMyL~C{RqJeb7)>1=ZEa5zX?*mvDrX z{=K?fS%cl*?Ynqn2!y8Y9XXg)sbjV+yN|Yd>_QVFz zE(zuLvSyMd5roF<+a~xT7lo~1Bd`=hJCViEbY;DdjaH8A!n&{=_7RlQ1O!cuL&?^H zMR){mt%z4Nk??~jw_4eDA$UOu1yoZl5af8X8w#SAUXX@lfdpuW$J7r&&TQ>biWAX@ z$?qssfFGf7N?@>&+BQ`xj|UC|rPd#N;zfRu(!?X$Vp@oyy2~L{N7%-<$|9M)ep3~0p^bw&zELSn6kq*8mxlXJmEHadP0ep?u(=RyEpKO3^%m2#qjB5oy#^@ zdK8AF1IR6e1pmeA7XJ&ZE@aLFv=;iG z;kx<1aGn3(x^TrN9GrvxMdNTR{x7Qkr#$cf0NsJ`fB#Q-{~rMU9}7fo_13#8}ah37l`SAl6RmhUp~OU=~n#1G`(;&#lt$EAwL zb2O9sx}b?v9-?_BHN zzrW;TW$>0QDTBTeS<6B+ZOVepBh#jFA+OxxmTDuQ{UXA8J~Nb^N7pIP8N%?xnWznC zm%rn0rm2J!AexWUbAZRQa(@B9lN7bODA>`!tkNUOzEERziBzmiG?KiCQ(NLEUB8mx z;R=aDYp&M{GyQnEJIN5ZOw9x|Q(_MgD|*j3R1U(3fGRuW1}Bx%L(3VJe}Wi`qkcPc z)O9XylCg*bC$0JV$s4Pf#0BNhR-`amj(tVrUgPB(YU=u0ik&7T1QWS_l9D}{F;2GX{*;XSCnG!2f zE2e>r4~5BpLRkdCOB;o+pofAyY^iMb-mp|kV9ttuD*IR>NcEL)$7ibu-fK;1GHm?3 zHfdlt-6S;#j+J54qN7$C0pCTk`=GlC(ZSnv^vA^VJ5DR2&hAS&7Rt0~HAH=$aN(}j z)(1+su_81Mgy!L!*q*}%X0Yp|A;yrcH$=1(IByJt&f z<;@|nDZ=3xDun@G9hOu#24DWU(LPNgPTGM*UE+bJX0>ofm2Tq~K&_r64dk*jzbRFE z5bQ#EO{_#4<4n;Dghc#aX51yNJ*rHrtgO_5Dn5PIfO%L0rlpy6Q>}5|o_*O_p=$;> zqjO(HfOa*Ql~xWxkTCk(wfc(2vII>_n68( z>G6Ik{#w{L&wL-U;={wkzA2%7oLVnW$BG3okf1MqN@&?pkis|n$1X4pz$d#J+dv>x z`pM*5s*vO1#Ra?q)YzoEllt#_ncPfl@8f>6I+~I(BzN|K542A9VsG1J65D**#>eQjl>Cpn1eCmJ;H z=>qEiWhQxedp?0SeFXQf_mgNtf1adY3lTX;5)aiKFyUy2Rm2L!-GvH5-!xfwYJdLx z`HxoG(X9PV9$bQFX!bNnz8m4~d0 z<`gBgwUhVvX>Sq_Z=C{4n?Xe{Z2bk$O$VKS?Vj6pA2?IqZwq0hC^lP?(+uS=il;aX zn*x*&By{h{oA){_Z>!iWA401CetfqgNSv8Ig|GEh;-ozi0s9(OW;bN@1H%>z~}i$T1iXq z@ubW6B$&h0i}0yP9+R>KljH9c#@zGs+6;GL6BQTMT>YaPs1o_n!6$p}F+%IV_t3z( zqZ`xir+UAuSGOL$Tx1^n6md8kuKEyR{kkLuw2R9*8AUt39h(W@bFUT!)J|-?)x?@2 zW+(9PV>fM}*h~GBl~378ja_gO>}VQ@dp8DNbK@ZT%|GgG*~%JIw)31qovNa4NV)@l zfi}A=z9njajxoOceP5ck*4#`tRQ3{~iB2rw>jB*Xx~wOyWb6L?tO7I-{(nh zYa5QeEAK`l@t3EoOP9dMWiBr-uQ(8m(PMZg1C*?+6GzTveDfCjy_-Xwl5F?;o-c9h zF7Wq|K#ufU?2Ux!gIc6HGTgIO>gK+u^BSC5dZ=hLjXMN+b;tR~I;xsb_QX@Jl|Nx9 z%s`RbmMA@&*JcZE@S>T^G$7F5|M?D@;Fcrn=EnlF=lUkD@YkEg@SA&>uaiHYj~k11 zCt`#6P&tm%*5DHm0;>=lk2~}*)V-TNxIVaB|2x>`_xIuYpg~HPa#PgK9|zL6G=KN^ zYf1L*mzX{xyiZ&i9YI5(&iYp#Y!r@ss7U6HRz-+9q9uVEduKip6 zM&LI`2ga#_65|J|CUW7f51C=TnKSg(WMSs^h>+#w^KVFoUT+D2ELxrd-ny*>yPR@Z z#se{(HG1$zXZ0M16{m4)?|kuERJh#l+zT{AKMG8oGO|mZ@ zWhfMxw9J`_n67On<4Va#8WY#Ha5I(4>Wj6rT=GPbqtznTY!|Zv6YDAAUTS_56BXix^cT)0i z_wT=j9ZvHACkpqYYfqU@5+l-3WKUmYhk`T{l8^S|!w%=94^p?X738LJ%yWLsyrD4r z<=K{?0z}P6_Q)EP37>IQ@a(4mSfz=6)wmcck|*Br6Aas3@Yeh=^mn-!1{2m{aj3m} z+jcYd=0?7-oUB70l@l}0v!qy6@ZqHALPFte!!YAgI)AV+nxdG_%}`M*ubmy+ZjV`3 zdg&`@z#yemHa!$#0JLB4t6`P2Bz^EM0zA%0P~z;^8OPmdsk)YEV!)gekC-ZUXjqGR zeiX-Ji2_&#tG#*6)Rx&^JH%tqD(Tb#tIM;^rtB^$`Wq8q@s9VUYVpm+BRikA(#GGf zVsGc-k0RCwzI$|nANP2I-y6ga(~n(Py`qlk8wr~(o*kj3%Y)H1 zwxt9j1@hO}-hXf(OAlRYlpu*WjzI`LD1XLes$pr*hKIOrH@Wp<=q0v(#<>j04TSjP z2NCfKzbmZiSC>1{mfns?p?I4C@)3RMT>_FVH*m%l=6|nUvv4KkhAOME-c)^FuGpwTn}cTD+g?|`CT{f|kDeA%CPX1Zqb?YA6uZ2J?2c*yd; zW->Xul5gEV$zk-=xp5UAQVOrtsu*lyCs|3FI(LLn_lUVPVG+-BanK;h;NomH@v^WH zM1hRW80^svRZ!g)vOgN3p#73nWd9r($}w(I&-zX%NLb_7{DI~DYgv{qSwmLp9p`UIfNADR{Bg0w zNcmV%VMwt!DbBI`RS89CR7+m2VWV`g3l z*9=={h;S(|IU}MICF~gCn z@@&i$S!yGRnxfDFz6{udpLOhO=hL&ZZh6gO=UKU!_eHZuU!}jhC%#7`LVsppa)$Ros>OOwTy$ zqrJ~49h5)D*~P`#5#o}r!a%$_73#3Mjf^a;bJ^~;MtDs(5d>$nXTTY*jDQX#B zaJ2ONK#VFhh+M8I!df7H<7%%vFNJ=cAMbT4LIORZdIK4h#a0b!jt3hDjEI8O7k77= zM|Iv@F_asWyPQkQ1Qz(`Kwh8K+@>Fvg%PedLk7U3It*#r8pCILR{@U6NA8ZH0SyLX zJV&FC;8<4sG>K94X`g9-KTGT{L(xD%Q~T}vVqR|NTlr-lEJ0sxY0`0>_dNG+*}ko_f7&^&Q0BEBloPe#kzZBZOc| z=P*SAha-aWO}t*#z!%UBwO8^Idc7*gSY18z?k-kNEJGI8E9Iu5BkZESOPz~8BWkIP zf6(qtp2ND|!D@=9v(S{hV-RSU_^Lp}-19XQp?|x2h}{{DYs`Lqu&nVQ3uw;LSsv_Yl`4T-jyABTB>>lYjT#d;kd1;>k8= z8e9cL*8FTJB<;ie$6Z8*oC_fsV+V5Mn26omArUK#76ojEIDUW=jLVO=bXo2hA7^M?S3!3v z<EpDTk5fFlHIbM%r>X$Xdnmzk&&Oy-uV)ryfXc`J;FFdnd zS>_bfP&ja6_awQL7I343P`8BUigKfgWd{#`?8}%-^(XOM`ZUlP{_fi@wwM*`P&Nx~3^x@MNTa)03g~&h$By> zz$q)@A2zsl-Bxj9b@dwxo&c&{dir8un5J8Ee3 zj~vm0T~Zj44H((zdi7bZ-u~^C7kh_X!;R6b$qI=Vc|b@?c2z-g=I*%&;s!fG8QUZX z!GQv-#+d5Kpx@zc2?!2H!Pvxx94R-dVg%H!!8+GmFSpF(lcg8bU&{v>)w zs(lNV6h?rx3q7>YBmtZJth$xbnCisF22{q1{eSbN!!v$(P>7e4pM3YT*o03Z<4OIcXgk;Ll z4gUF#ePu7)#H>r719j(E4@#GUM84Wy7-`I;8323BO`Nmm00i(puIllFSfFa}uzEEh zp}+M#M9goghnoO+Ee}Ceos1sS`=lol+DQ&en}R0fW2~jQS$6JJ2!`Eu?v@*}jRmU# z9$ONJnE)}FDKKpnp%lSihdZLkRhq6P-_)io(0erDrr{slYC1w3lSDgEt@#zLzD`%^ z+k~Q&-ROCTP|1*a6OlNjJEu66?i^DaF7U?5&#F#ir%CX4xkT7*6-kj-^oni<(iDGp z!aSv29!xcD@C%5K5_?YF#xb5XN)@3PK6ET)$cLuZ&>te-b;{rjdecFRr)e4_FOk;8 z-kqF&2m(FRe2PBUy*=p4B&u~FaMVgg+bDWF3M z%CJQMZFQ)pkU)fX3;iQOlk9q>tpdNr*dS%G2Nn-t6V|v+(q@uU$bos|5(M;n0Blow z!--vTSCP{Wv6W7F8>+>kL-`S+cpKyfe>9(Nrk<#He70JE}z3cB-VkP3CmJEJz(q1 ztpJE_u;i?U@u?Tewnxh8Si8kZKNxOyc77wgi|!-*wtw4VsCFR;@l^(>=HZ8#Q&x%sY;L7G zE#AZfW;dyV*H&o%p{*u+Curk~5dVHS;9h5Y4jqWSwR^AVf_DiF+$Glj8C}jjslxR<#_3XyN8#Poeb4Uf53zmD1Ji(6p zTtA6jp>g&lG(>tc5(xl!jdqX@5Q1^pC2CVI2H zNzfLNE))e4Ri`ck14gl;Zpu}!Vvrv&@Un}B`z$Zd&-adwjP!!feyK)@IMt42ar^;< zxB&&V6~S?dk=q(<>z`Xhh_y#QHh+)h?;=HB!kPQHcPu=(YkZn#KxQlKO!d`=O1?m_ z?Q?+NQ-uLpcXwBpnmqjpZ6vz%(tGK^ySoVb)AZf-&yp10$RI`;H>>&^M zw&OWB#hIy@EO(~z*Fp`}CL$83Kk($Q#waXIC6r~pnUWkkk6nE0KOa(*aHifqreolQ zaz}0$O07mL825X}eE>NYKh`uvfi5#f552u;J(q}bAB!%IYJ5be1Y;=poZ{S z-xuOlBXLVRr|V+~9SnBV%pJLwS!uw{Sls28@(ls{W)UgA(v~V7#+qQ;l-U}a%;D&x zm*uPiu{pd;60&S65Zw|(AFGa4vHM$f^x>Kl4~O@Fj;$xNCg9t{jr}{N+I9REi8uK) zkP{TEe75-e;~0MTbmj-~_VGa(@uDR*^|;bY{7Wl*0-#;(;p3K!{BiP5I}cd|XXs(- zndrU_l)PCvC33W|8qrXc#MH%^32G;pJGoQpuzS2!oAiL8lmN02Y|+=u0M+Z~#8(~o z-rwv$$(%eHB|sKZ&xkC<^l|~(Hm-8JmtXZp@)|@(cVn03Kl{30d3@zzk!kIoURcaY zlxzl>uZ*0|Hbyq`j<3{RYOA&VFv?jJf%2A2hlGSsBcjfDfbmrGxTJT!l{YXf>jm2aTzH`KWZb5vOs+ ztu}Go*=wsBCP)>>9QZ$=l$H{AV}hX;qHnBI)2rs^^(QWt=+fy$8_|^tTRzkh@vtNt zyo5$3BWEaoeHOeY5PfwUyZGwm;Un6`1UcQdcM2L+4EAxjbp152g?KtuUvvvoq`Zgg_Q&Y{&!kE79&& ztt_(7G}!|@GYx!GQ_c}4W5U-G3S9mD&zd8}6Ib*xzQgF?GalslWKN}>FTlhDrJ+@ z0shsohO!+C%7N>b<$0GJP*(@%@Iz$_Q>Fre=zA5+w1(J5aT;<1CwKq)&lD$Ze)_@m zH&jYi=ex)g@{O1gj3I>Ma0=KDtZ4z4Tx@y1SY7PE{E6RoJ-ztqR+oXZ6h?Ql)OVuy zVwJ`yclqYYp*ed%&aHaZ``X~?w|gVWP3`5TQO8=nH$3)eL!2BR$Mcn-dn;Jb6IEo% zbjV~_2&x?8p!}9jduqHk2PE4g`#~&gvVNKBRH9g@`KtEry+kO>1jQhUy9^+RT~gdWmC{U)?EtGz)-7juD;qPrt6w zPN^%&-*kNVq+!k(iR+&MT`Z;OaiEctfCmfKPqAnmM@u$K-2~N_OLbuuIs>056=MhL zZF6uMYbIeFL*V5+ELlg*J=c{Rr|{8*yj3+O(pIK0-}&(*nA*b(?$!ba!3I!yYk;Ai z%x8u3674-2cf~j5zDto;35MLpVkMfys_&gEiPAysrJjU3Nnk)IZF>`rogz;j=t|G5 zf02{&errndlvK*eC2suf)-QTfJKkJ{#cw>3z|qp!7Q@=w031KgQdzAAZ2tHyexv@5 zD*dL(K*2a;W7@4BE?n)>5D#$#oS);JO{oeTve+iW5(EM&Gu+jEz0f(kHD$D)m{Hch z`=_5js1|IvbNsejb7fOk?_YWU_{O3kO{{i@R8j#*Qj@>E{%qZ{ z!#+|RKi`Cx&G;v-AFmEjFMdUvXFR@Jns8OQmtYN^Wz%b8&B$)UJOdmDg zcyO8{zJFKZofFkeVNuzxJOWvYRRx(aALmYZ_)i(cPY`Ov{p$Dl`uMKf3;33sKPyJ zgmD6mqzl7L?4rXDe%_P-ar()9p+UtJQ+;vBeKdP;8T z_N>Jv)j~m{$UC-Vkoa7bPQU~x=gMPjtRgDg44h1n|L#XZiyqV5yD~N>3olD_NT}D9 zZ?w4nQo6`lNrlG~NGA{No&@o?QyKYCToM0x@hJVE*oqmM-J{7al8!krgZ%! zpsF1cbnGg2$iA%1D6DXzE&=5n9@U2 z?F+#BV1vW01?WO?DVqf%Q-XIlx0dYmxSoD>k^a0mxpCj}R{)k{_g8p?*nfD}U=sgO zu+1r^D=Y~SAJ)C|J|D)Ri@)6d^Hyi!J&Mol!RZOI41`M`7rF}}qK8B;D!Shtc(UG73dVh7X2(AyDvd2o=8o$bSW8uQh9j^j2k0^SFpVtIEqp>%4f|8SNbReAi}5GCDT?DR zsQHIb@qYGSHo|Np(Y8wFaRq&3cszn@=wj!6(r-c#Dl^Ak z;lPL-w!Op~YXVWbanFp)gY5wRI2UbARYS^!Ml9NiGmyb;7fQQmyu z`nZ^;p8{d*GO5csg1L7XrE)$5b<&1hD7!99t0PLU=`llVP_cn{KO#XpQa{j4SmXPN zE*v4UQ>#u(#BCEh`xfj0q?F@zP8G6%t9fe_xgq~B;n$?Dbc4p>(Oemz>pHI_)Jk*# z0qsu|OB^~QY_@{NY2?sDpLc_C0uMmE@a#4AlSBBeo=*TUo&Y~SS(##%M!-}bxow9f z1L-hy@hEZJnsMU3hlaFaExv0N? zL0J`ZG5+o;7e|)N1{#QH`%H$0jQ@&Qn876K!u<9AwFcTguB#)Tq58v`^rE9@^4neQ zwa)h{628P7GoxU<*Jg-EV%x`VC4bxsl$}>4&@GKx#UWbG2@Ne9$t=~cAJBJNJf~L( z&+V3AcB6E^$zJz$#W2oC2kE=IMKd?w6LATfUa1^dCztTEW!t=YkZSGoLEl(V{p})h z>g-QAQ&OG`>O`Ov(QWp2=*(g|j2c3S9I*98MX&(cBt9N_N^M?U*A^y&P#)Z7#=*u& zG8Fc97@#6=r@RH#BiE3dS1%R23D6M*-zOSNh*J-R0_L%?_pb}|(27OrhQ&d|Me7kI zchTsHw@p36eZL>1!(KJL5q=pg2aSShkK>w-$`sy(NYp3V|E*`WSW+n%s&^zp1c;`) znP0T5q7QEFt$&Vau5m!Q*Vkfer22!DkpJU#X3hDyX!+^mnE3(#j-`; zy2R@Fm#OJT_b**D$m0sB+`R=c#(Q$8Q-;j)G~MUM?H0sV13pOElP1WAp7u7$HU!aok^HZxC7}BXi_r>Q)k(m4jkEWGf4p_*%Fx~z`SWM7X-evx5`7f zlt99`xgW2DC8s`hT6h|$WU0sOIQ-O)PxH{#s}UA6nrG_)v-Te6MT^*E^7)Thj*bT% zgXz9%&ii~-7_}G}BH8&OW~%QYlU#4~jd^1I2IMElz^*q+126;os*78WYlvd;eshXw zhoor4+IDx)G3AvWoSaO5G2F1G(Ed{U8d_yvU#XIq@a76)YrdWyH<3-t=Q+=Lz$f)o z60f@TII!i}m7zXb$r01RQFYVitS8^l_l`myg$~0@tF*U?0$!L;QVop)cBpO@g zCm;dF3?G;F1nU}soQF&3hPmG7lVNQ#!KXE;2d*o}+6AN+qi%5%Uk*+9b6dhjdFtbq z+BqPhdWL{X6d4$6ks#FLNazO-=kvR2m2L9IKnfsQr)5#j^d_H0eGv7n{I6yyY9r667r}A0&Ef882b2h5# z%8KWK9}Iw}SjD^1v`!cgg6rU`vek}$p-!_30TkhuY;dyE^)rVC$*wRR{gT(>+-*K6 zEC>F!+NxoZFTgG1WIX;fi?FyWtlqHvNy3Cw`Rd12(G0CeYRHyj?x>=}FT!*;Yo0(} zZMHHDfq+jV#}_&$G6}h2Gr=gXfZJG{*dy;=T{hCh-)<(lF)-lx{qFY|3jxaznG+K< z=&c;+SD))@RB;ApE4>)|JYh#(?L>fuL`VRY&RVN0hCWqAFEd(xye^SBtRVpgH0p1?WH)fOc5LUJpn3c%go&zbYyu)oq!!0@TWR@e1+ZB> znTAT5W|O>E)>12rmH*9&Xx;@*Stv+I+9j7(xEa}4_yb$7SqNRVTnq=%v1q1PEz)-T zZ(=ED|7>SF`~CVYbWCFUR-$NIdu6h&5gFeiP`qB)IXNWF;GL_A9nJKQemgeqgJ&@y12(4{@8rfZia_jZy?l-abbTH>Tv;ndt7G;m9jNXMAvd;$(ElY zX=Q-V=%vZ4B$xOLd_D?-CQ=o1rEIyOjh!=g2>pM^kdObw08CS{hd8oV8ke&|igvFS zM%rXM@Yk)}rR2pde1Q?pq?O2rM%(Sf=8)7@DiITj;m1=N8Llt}KfXX(A8b+6=;|)& zS1#NxsN+(xWFhuSKgc`sllD@u&9U{dZ)!?fZobuy*6(njn}ZRYwLc5fgy!8~9nh2W zbfIMyyt?^{4>Nov0sV|lA0&d`8<0$H_PxDDNPf6;VdxCWqm>tUODGFqLDVKkDUYnk zXFCy=T{8NMEq8x7Gf+p(=D-7S>(bUhkRbmVYW5-F$3yuY%XEBD`n4};C=13q`M}|= zre=r+8;}*qvY_LSY*U+}gQoeQRr8arCuJ)^DhPzuocqmV^AO$Q>`~AoJ%n)ir*}@$Qm*XZZEW+WTB!E#L`py1XZJ$t7Rq*jHr%Q3@Uc*1rpx)M46h13s}-4SQ1(fM(P}o>BSAMoWGv#=kR~tI{=eMr7iR zTat&*;}r6hMoqNTP{K({lp8C*H~|$3JT>9i&|)*iEb(x*M7g!HOOkt5S6;uWvJv{s z7vfnXj``Y3%Kp;61-i!8=)K$O*Qav6J#Be;uOTU^ldN9l!O7z6sp=$aO76kh$h>g0 zA&iO&jo$dq$%2=#u3szptTT^?2gik^Gg-=bSdSj%$k9dsuYq_9KOZ&smwr(ipKKN8 zn?E-gf^TQBk?gMgQ7(WgPVJW{lHUESHL z*({ICQrf>yEtEqS1rh{@n5cJq^Jql`3BSn1X+EZn^V|rRHu53m zu3wkbaHl!Gi{PR)(Q9ftqBEVxPGf=}zsspDr|f@G?Y7*vC6Nlhk%AOF3w*QWD`@(h zgu!bs>O&wjN|#N6V=Xo$XyPb0#xR2Jip_*BMu0-mC@FB?OJKzi^!Q`QdyK)DgM~AR zCf1b~eszKzZH*xXg@DH4G8ZAkTglkGK+5nss|4yE_-!>2f-a+@6>6A0<@$6KX__{Q zmzS5C*#uCVnx&ycTr%fB(u}KH9y3I}UI6uZL$&j*>{@7r} z7y_FF2DJM6dX9t71E^h*Yj`ndxspkW6wB~lJu#IcF6)N5*sZNLb`SB`V^LJwdY`Z? z{t|b0T<`E*j+G7gG$X1tPFusY?dE7~GkbAlQ=2E{P+oLo6#2AWe#cZv7Jayw*;cjZ zzS8||7a>mdZyR;5Y>=k$XzX`_;f4>WZ!sW+D{tl|FTUGJs|oi>ob62))X2}Ew<^Ou zrinsSh@;cGIK-T@ouY!8chZn({=u*3Wt;uKiobN!4s8-?Dg-P&*D8)!NgFsy;U02v5Pth z^h3;@3#+Lr#d5~kb!{7*K60Dt0tkH`HwhQ1;uyUeC1e_icyL@w?o}d1Ezb#>^H%Hw ze}PU}&jFTfaA4%~uL~e_iJ(wxxXfWMRxHP0z!k!SA14w580^s8XL1(A9I@>XfEGJU zgNP3i>Y25MhO@@EGzaSxJ`wk`&Jb9sV%IzILNh4y83xK5QWa|%3)>xwo-v6J4GuRm z$TTyu#%h@1X)TA>T8X%O3p2}Cur%3- z!|x6@OSfY)fDPKfcm1c^+lOh9Qg{RXm~J^x)f|Qm?T*gQmClZi&e^|hF4cn0nIR%X z!ZcuhJ|G(Q#pvql=<@1bD{Clz(S)pCE#xl?x7t1MYa~x-#?ek~__?^N0QI2?3Y4ud zLLCn@cyYON+E*J(HEogihcRCR5t57mRk@KZK%zv^pDs}2Iusum++DbXn0}OyoNv9A z)hJKKYU-bH^%EDDl$`VUvIjph$C+>*vN+xZ!gIqf~7-=gqx_KOIy)02dLKaTquMq1cWaK-70!lpF@+KkQ(7GMlf9>V5ne zJ&kWMBOKie5b}`*#liD^LjZmFj05FY#N-}~wsD$d4AE8y|Kwj&Z2688#exK22HWsH zc%SHJ88Uu>&8A?IW(Ltt`X>=xJ|0Wj2~sv*lNH5t=(er6VjbJ{9HM4oLp+j7Tb8lo zukI!0k*mq72t#s)pU+b0t6I_zSIvf)-#M_bT3s(-T>9k3QPW0rE&kdO$udm7C51aoPiF zP|nm<#mg9O;WzbpH#~mhmiSx(KTYhu;+=a3ylf@zLowa#qf2Z5Aco%ZE1hrBU&q%X z9*GreSusiUzOBL*kFCnp!Q1Z&HT-{{UK~4TsDoGj6ljDN1Lu-pX41vv9yJ!W)(zH1 zs9DA{J26S@q}!eJgXNfR0fBTBtBCgHsnU=2e@Z^7~{rShn z>mMVye;xAvXYCK%j_m(&HMRdP{rL}Xtp9M5`rnhJ{I}-+>yl^imB;?y`9|>y!oAu4 zyCmfwTg3leNs8b<`}$AcC|)5!?*FPt`JfuR-9iztT_-#Jsvp?|-_76;IkWJ#MC7cL z+fwk2)XjJJ!mG6gZnk59-bB;yWhDuHI?^49;#OuPLu>BC@D0Ou`66-Zs?(lE!uRj% zUmG9Rk23EEms%R?i9f$Pi@Y8|MX-#w>O)2#93Qhlph4)#%N#%uJYL;Ce&nT7wN2F^x@msny+0gQgSzb?X~=)(MLG zck*P;FU3DNm=J;a+%%=KW5O7+lfB&)_*stp)4Y6Qr50OA{#ZOe# z(BEyC(kx13LjLYd8a2+19vk(RT>^koeJM7YVu>krLQNJNVu==Y>&IqfQqUB}E~Zvl z|9!X~6lqx`?KCz}N4jh24Se&NmC@bY!Hj@$Jy8YKfoQHo9ys{L68|lGNXnEFFLo*p zgYHhSHc8v81

fnDDUQ zkSCh@Qb1btp`AG&L9Oz*p90~&Up+Xeez;uV99!lC5(2UM_6Rbkmv~S2*^Gz*g5*9J zv$%4^LTl)uOL+sCSsJD;WFq^H0ijADGhlWTF;n`9NG|^3gR&Q{`IJ9`GRyK}`U}%1 zhBu$R_sf_<r}PG48`?p?d~s$O^BtGaqs^;!Z@*2O=t zXo3qE6uRA--%i@qRC729W|yx3lBv|c@ANSPymu(rWPL{AA`l-yql^P~@~Hi~y>d|) z)Rh$?hS#s%0J|>;Yfk^-vuKSLGa}|sNHn;*@D)IBEyhuRT>4-h!K2zmab*=m0|sJ< zpdc{B=-{h5;XFMDH5#>30+lg&s6HSQSthB9JB+x1A4mn3x2e=hcA2NVs-_?>nL8(& z#=R8x0!|$d#G8DIz-o;N5~wwqG~knq{dB!NER!Yih`X~|;XO%u7R6g3wxOjY>-vI* zzwI}9-XG35ascKL@&eP@G&kNU6$IT~;UXNtD~98hGVj4BkS2Y?kvncT*_4Yv+}&&` zE#U7M5`9)LI4dC`0!o~xFu{oDdQc&aDprfI_&Z?4OdgeUw~rt1vSUZ-)c0No3W)|7 z>fFER$u=3bTr>iPgWn?ns-3r>844wQW6IwZE)s>S9R8`d4;KZSI~JOyqJ;TzuS522 z2+M}Byw)OS6OCw6d@F}yoQ}{O(`Fl4CKa zSH%|Ybnx8|uZsL#e;v?od#S-W30sj7L14k~ka!`{?)#ZClyrohUVvs=E-?y?nsTmT zdm!gF1GN|hQD|tRh0dh4Krn|rX?8fP#4@_0yt4w>zWFFXZ$&Z5SL{wFH(By8WvW(I zViBlhz}@O#0I@ya2GSD2*)*Fd`o}JaRBUQ%yB5askB2G`B(4hG;uT9UVo=Y$9zWMg zm9crmEG~#kor-23E_kg)6UZv>OwkF&0b7^WG=d~jH$Xzq`QsmAjX~sbf%2r;?sOxp zYiWoO1h&m1mF5wOtV=``05%?p6>f2#bo)j?}QOrx0_Ner985qcp(mOJ>dkkd|v_{l3C&;f^nvklBC@E$h~~m&=JyuRETi zk&-A@V5ihthjURLM38#4&5EFA_NWL6(hV6!*4md}M{X^C6M%}(&T%}l^%aCm~mrGo%pW{V$nou=ulF2L(;y54GfiRp)I8YPw> zRHJCO>f3hBSzv;vmr8sD%u|MYW$)^N!Dx%6$9zSk1ZS>}p1LpNYF7eZV_{nbm%N1$ zZ*=)B8Xn84Qd{BJRfJewjB*wsFN2xhRUC0}k_TRweKQEi^r5OCif53t)TKT-`tdmC zh~@Zm6)?uTl^YKn8~x&Q9?QK_IFqg5H4rfIfINjv>+iI;3HYd&@4@xw_Jw+88x}R&t}2Ljv5L&{G_FdB@nXSesK>?n%WwM&vsezjk?Rs_X;%1dfC{u`HZEROmv3ENnU5;+cFK<`tR-K&b?kQ<@hFnpl{aaonc1^M=;KzX}QMtEx^CQg@3Z%|lj`g7yb!BA*Ia~fRBge`7Hb}G6)Pb*~XVs2=xAS!_W|9aBTO7|K1_5Unb zEd4+KhyFXx_y6vs|Bt@@n)EY$#=rjeNq-i1Ik9L;NY3zNkV!+XS@Us(NyTS^1lt~= z`n5&yBtg}l^s)38s1W9U;|gP+?&Z2B2l<%VMJ93(BHcmA#r6u4VN7Jh@oDmU&c(3T zu!_+#%`J?Ec}pjsEw9tNaHOM5HUx+;=tC_igRp`7>0t1%@1(VG`K1tNktsFl3H>}~ z5My7(&jP_ICQy#;LBfH!nPamM_ytxkR zO4-hW!B1VBsK`j_C|5;9S-wVZ%#`*wupxE8E+N$|1LNplyVDayZ`co*P|H*iqTV~H z$qX65-328=sGs>)?KPv4ZQ42=7*Rd=$v)XIpnB$BZjU<${ptv!+-ODNFCfeG8Qq{{ zZcZ~wC8>wv!fC`e_Dw61bX*10OzY3lrnC)af5w zB-?`%z=9x~Zv5gic<5n$ijm;k(wZu~7sT z!!RP5ZpHD*>!OG&<~9b0k|9~WU@x8zPmd?71L`3IIqG1CXd1*nr6o?|B(85DBNYKj zzWlju6-dB1<`v}0}kU)P$LJrBPE>nxg+g4bgVEBjGt}~ABMQsT*yIWw9)}~ z0gNk)A1V&Cbc#GIaXq}utrB+QcC-!MaPwy|P9<#=el5&ZQJh|l0AovG;AP}zrokI7|K9x2KJmBc&(M(tYr-v2T6lxt$CXh&wuGDdl z?iYQt_M6+O<0H2qCjJ=iwcu+UeX#}u(p;`RqKyHM0!(Q=4W@Q4omo8(<*yGd$CEjv zu2+zZ4=9t+KojKW`#~heACPC(3ssIxjLvFw1=}Bm;rqrE<--9~1mO1g)oh6Ujk(=p z$0jTuALOQwFl=*@GVj^0XAGlZf&lDmL-<-*EQSVAKuL)iIO~N5G>dCn`ppC@Bmhya zlE2#^g4srjix~{#QHA=IfFdek(iOV}&R7{;k~242|GpnRc(RYgjz*HFU+5yxkfKHwTGx}^J&^YEE!ctwPJBB+byrF6$1rfxo4leE#rNC=OzI9d= zA^!}5?6>3w?P7+;%IZ0!LIT})uJlm{Kt1-#HHHsw1l*Bms|<3H8!7lHBNct0ePIaz zLQFg=JY#H$3shq%e<{bJ_CWE5(~#TIN(^2h!6q-4cF4z_kq1x9BzJN*2Op8I3L(j7 zv4LTrfHGtzo!<=AQ&o9f<76WQn~c=~O1{%JX1^K{?~ct5NiSFkwUXIFB*?AE0@Bv_ zIiu#o0&$hNCNNW9qD8`o?GdR!aw%>W=O=&iz*qwU9aIxQSHhHsSR-X*7B?Jc^4f+( zBwY$;{<6srW$aUk^;|*#QRre|bu$l*tYj;M<(Wdy zHs5d2Ztsx}M&7enDuJw$DjlxObzVr~o5Fvo*q`(R7zcD1E=;`bE~pi`gmSngO zkTAErDOpR#L2%7eB|ZOPd~g1uuXO!#c3;9ZAb@GZ(WlFij$0SemUvq&A`p#C0}qT# zqKOPn`@6_6{cPH6Lp&;zb&#=~mZyq~A|$^1#gA)+exIDQBx;mU(z@Rt>y2XC`s_1I}w6G4+fq0pmc_v z5sum*qR?-jfhqR~HKW&LM55=TV#t#jv;xK@;F@hMqb^V6nwdJxsLBG@1pf}%<-CG< zvk0S*k>A#Qsj^!S&1{u?#YYxQ@WCaEtoPiR;kPX_50Qp4!>f2i&b%SA40mcB?L-`% zwxIiUK{@w3tr zAykwtX1`iU16_0T@WZcvmTD+q4rqjZ2?y6)GRm_tZ+4bKtRr)zg zf-u=tB@{L%EUw{*&214` zlYSfyUAEP&FEi5?EM1J1IHk^TSc5bJ>u+49Gy_SSfSCoUl{Sc~?5k%_WAnG498t3noV&Mdw_yNfj}m9yx>^*ZtifG)6J zNJ+Jeh6r5nTQa(kQMKaA7)b{x!kR{K2h2r9pNTbJudkQPR$QXZ^IWc;9!f83_ok%H z{uF|!@x4P^ZXth|-pLOs`}Wfc7GX&|GhzdL>YJeTy1(&8{T9qg_mS5b!Hx(NbmX)D zEXROXZHH&8At^@=VYR~mWu#4CFG*Cr;g$;!t0Nvn)5uqOoYgOxd}OfU4ZCVSgt5N& z`fPHc2maNZI%E%oT8|kc6pFBOWkG0LCSSPS@6sJZONACtFD{}BZjnA@&##W{obJ5n zb}aJ6D~H6?@NTAS1%G});(7W|!Tlu?DhSSM2>2y^mq^$Pe228)9nCn#rVsW& zjCt2e!IhIS2{O8Ueq~yFa!!$MuCOaBdEoEvL0FzwRLm%0-TzQ?PICFNO*jE?@k`f@ zYo|`VqA_6NmhN;ljUdtLuG6_6fcC|~52+-mwm5OgO|`BcHJOJ>{?tc(OH7up8Flj! zxC^tbFJ`$tD{EY6=P~-bxsxR=YhPb2Jy^>a-o;5w-9y&}a?qEX7mw7XKE8g<2#L1N zyC4oLM_U!OxwChlyUF!K9UE!fyT-Yape+(J7_P5_^*)V1>m4|a2L?>fWeyv^%PJ>N z6vj$tg&jZ48~R$nUL$dHxrVH3G;ov6M>d~!qZY8n>`{&I<%JAo5JETXys~5l+BK~z zL)GS4`J~DnZ;S?-N;DlR&(M4=QU@r&yZ53}(%@@6amoz5n$BP?$2`fpTHn2{)~fs{ ztQTuu2xnampimDNDcjWK!pGz`1?yxAgKmfeSsDL{`ESX#$=kf?gxBHY6Saa-(wO7NU-{Zz7*;tNS@zW8A2!#@Y4l~l1P>~%pR6e7;|MLU3cx{NzdVG48jocC$n#M-LRe8P$i z;k(7cYxF&~V$9)bkGv(!Mx&0)*5{58fnMVy-+*Ba$Ju3+a{q&2kQ4D!l6KOC#v-Z* zf$vU*H&z_If+%(-Y~A$DHjsL`_eUGj0P1Vc;SX_)Cj8?NhWwWvC3IJo+fwl2EWy0Ecy zcM%j0DrCK&V$E9Jr@iNEge-mKfQVo)Im*&tlCKhQy>IvssbOTn{Q9MNA8FJe5^31+ z%IOSf)lhQpKYi1Jthr0%uD$o@AQZP5uA^@}qJT-Pph{3%7qn^B(aCov*qh}HIV=P< z3>>djnZslRlj&3n4pGls>$t3E-qm-KRNYMlNGsYprafE1iptd)g0|+Er@>V=8gd~X z7xQQf!PDu~f&zs3w!V5JPkwL4gwQ2WIgPMIN z*Tt@vS?jH?w~e2%>DL<$ltk-J{=(c zDdW?n;_|zT-*4gb^1np>nG@^Z-sQhirI{F;n9$KNG1Aeod>&bhKMTueVgD?jIlx$$ zKg;jB-`781GZ@j)G5>!3yRiLP_q!eI?{c{WmQ?5&9doKQ#Q*^fxkpAo3TAzw3V^#!B~3t^XO9 z^FOs_{GYV`=Ysxktv^?_|G_qTmcP~fi*0|y@VBC$p*;T=w*Avx_=m{9+--le?Qa=> zvh818{7dAYSpI`;p9p;R^2fG=|9k8{eRqE{@h?1CKbgb&8_z$E|Aiwx-9HWR-*IIA zpV&iB_n#*Axyj(4IMV$mef}cMU-bElYX5@ZPy75g2>!$1{$)gesq@<)SbiJIUqD@^So;ISznJm|$p09ZSpJ0V-(dO=EBiE(|FW*% z>VJ+Ef2jA{%6_Z(&#~#V?H}XNUy%K6m486@8@As}`D;x2UB~>}0{^zi&x$|l{t4gj zw!h17_!$2Q-9If=#N5is*a4qb#7f`ESjgDW*2wtpO(yjC^q(K;-)9V;BQqW2ryKiE zXY_AA{Jk0F(=E;XZ;t8JN;hMrO!*XQ5LZ|69n|%JHc2}O5K2h$07dxq06{xfsLut{oWwK&b`O5S<17p00|)~J z*8^(=0wm@s`8oP^4B)465;GtJK2}_xB?5&Esfr#x9|APU3R?g=fJq}PK)EkA0G}q7 z;#JNpwz^JjKQ`bXfG8(C{Obvr17N_}d&wr?86WJTp6Jk$DZny<-i4kOu=_b+KPo^Y zpa%Q5ckyjy8z?Q8bCMNgYfau#PQZj@c}5-6l5jWo?4!%(s6#d+G5JY?qNCn z7~RI{1~jrc)3X3|aPkUvA9()^!@L0TB)fpMwxZthe&6_Jd5_2LM-YQg$%ty?60?-X zH*9I;SOddwnj;@yPhSsOQ~xGtb7J%;o5eW1V;lJMV;s8A5^=D&kwv8nKrs%3p}dZI zpPAV2-FJ{->;q@y;|uaf4*aXHz~x8Gy6@3XUh3-)mz)nAjn$D+iIru{yCsOLDNe9l z@F&L+KmU(`^<_+}gI`mci~uk3WC!r>(3kze|7D2L$tJe4g{V@GaqEDgMZQ zrMs$D zvX;^fJ>v!Oo#!L_5~^zRaq(t3^TG9j;|f0v3L>PF)&L6mq)A#tDo6p&@FGHd-^#J9 z{|o30vW)r0!E4lNts^mD={e#vLO-yY#o$v8mbNMES$5+AhASAyRprlBB)*}Q!fYkz zEOs@@9~13ToIVv`pk^bXt2-BDa}ky`c<#$TTv$Yu7NBqNc18x%*>T`yZP2|+G9A6_ zN~_POJN&|x`&xWzgR!7^{2H`49nyzxOSbeF{ig9MOVopY7OjI|1+dP1}yI{=WCZnvUON)11v?kxp*aXJ}zHA3+|Ax91qWa%9Um z?M2f3!SeU4Ht`;@g6UU$k5sz9f5#&5roYG_z%b4AzCCs?JvH$5&?%T<2Jm0Li|}?j}?6}ai>GA!I!+9^d@5Pf=Sj1GSa^Et9PP!Gm zghJC)2Rx+uwS>oDDC7kPP=ca??1W%w{*|Sr7Yuoq*}D(!qj|O67{qZWsl$O=MIm$B z+fLN?YpBJW1l24!Gs~SV(DggQL@RrIkRZ;p2L^+Ot%AV={u3Z}!mxd8UJhBSawq`7 z2A&hUA>q#nRsMAT4B3nWw-+otvN=9WYR}13(^Kzzn8WFnNA2ge;UfS~NYHolV-Drn zkU4q{{SdDP(28QP8TN>^Ul}-wf_E^z$*~D-S(2+vgLB``9Ig!T=_w%esgGO5z<>6bB+r!Dq;he=|$ISyhqR?)8y|p zlD}Z(#ynaq+b-63i|izI;$hO=XMa;TH@JS_wSp0hw8VtTP8siLldjM$o*_)=j1%fs zs=<0b;iOl*xkspfNb6NwJ#MR-W)VKkCK+FVF)y^);=t&(hF5G%#D+HP`y6f|{D9tJ zF{=*7Y6DW-u{5}ovgReiC}eH#K0YKqKCR&L2=1cchs(w872?2;IC(4dBGWPgcX{yE zxYzM3jek?2Q!vbl%+<4Jh)L@XMmu1^RWP2|4DL!(d-*N=X zdc)7WwQvz#+njyJ9u*YRR^d4!6qNThO{Og&ko#+N=hLX3NEc`tWPUNqA~M8AaP(21 z@lv+P&Bl!Ol?_&7O-DBMC!L77@>KwxbRF zbLA=s0`97!c6DqBK4EYOc2uMAv>3p6R@s~@jH%JUV;<` z@o|;fhO^*s%?lk0e#_?isDOF-3Q=N++$F6^1T@MHy}v)r2O*t{2<#Kv-@(`%Qf z0NSph$|kqLJhpj5?oSW1$wuN^C~Ss{<-RmK%dl*Ojz&aB8N;xa-AZ^|lmDW{N~5~a zOAM4Mq|i`fM2Ax!5h#pJPvC+OZw0_`vG8tvIq1nYk8etsm`57QPIN%VnNjP?k6j;y ze!YuzJP|Y@9>vOCoukc?131{Io@UK}A?P6kE#1XK<_*N@{PX!93_rRijhs{D&`GaF zd+6fz!b$xc`Qm?)DNk`+4-TzO)^0~ya0O)pRpwN^;~ao0w9OuPCA&Ow0=$#8|&0@kU`9X?Ls}KWu)oc za6RQzLR9F07xFHGc`MAaPwJx#I0l?XUM1T%m#i;6dWLmQByWudcN<8iwPC`sonWdc z;aOkS;+T4bZ3Gk|%MK4y&4#AOcT7{Z_L8Nd&++JkDf$MgYn)*9MYB_ z37M~|1r$jW$V$fAF@l(Jp!erz_fF_g3o22ehJNxMwYiOX_fv;YX7n~maLfhTpFWm_ zPR6PpC8$Fk=|p5ej;rnLo%nPSDQxEsM5Idu#FS{8K}tXs`m`ZrDwej=yP!)I|id-S05Cd324sN-W?ov5F?VO$08{BrhBb9)8m$=}K+3BFfwqs}*oqbRcV4RwWskv6D2d1i4dZ*l6-0 zL@Gk|Iw`#@$vUrBlAtj5$t-30?nF6B@A0*BTdmgc%%49cr>cV!$zReB(8Bb#zWeJmY`x_lpu)qRC>vwk*RC1E&aDRiV!0KYTneK52Hm4Hy?o?P%9YU z4k0*aeiV}RHKp8$tY1wzMdWQD#Ocgp_Je4V`3-$%7L{lbup?wNH)qG{&EOl5q`Z|( zVkpU~EhXE1QXxL0+VRsz5JK*MWqTDCAG0;DH4|QXM|g#`L<*9APk^oj`gPreiB9{B z(={Y=c?_kjI$tAU@)j$nM=m3hDMc0ag=$W&?m=!38HBvmh=Pz-ywp!Qy{2HXGQT`c zq_%~Vxke}^&$S$ky0FZf*o#pKv z@m)R-HsK5_OF&XMRyZ7iP8MBkCDi)l*Nk}L)8d$wfRNvaA!Mim8(PUK<4!1puKj%f}pzMeq%V(FrJfK3$m9NP= zBRX=Ub%uyt%c2+nW|R7o#vgC>120uHM8uYWz)b98T^nnV9}W7o7FF_iE`NjpvTcQW zJJVSjK#bz1nKH(nYU^5hBdr&CVcZEd25-wq^Vc}{Hel?&>xI!;9*0@qgF>T_cB!(w z+;KQ9`R0ojj?zWveLL7MlGrbS4K-!?Loh1m$fF{tJkhtJJ}QsICYJ%Nn@-g%(*vPA zUg3`|b}JU9@-vi|yeIDdkoQHcIVB=(T?D!K7tz_5f_cBLjK@zH4NWgLPw=Or=6&!+ zp5{Rqt<*B(BtQuv_L!B&N!sA6(!}D*^$3U35P(kKTmpu^^;=exG731Me0`XFWQjvu2WiW-4 zv+H6$I7DZ_((9kAc ztU{%{=Aff3i7Z&F%uEag&cihil<~lu0CxS^H79_>2jK5vxi4g6ce&NBszzTL7J;Tf z8jY(;{%EP^SzX>xRzShKwJRih=)mf^>S1d-jG$3=CASoaTDorsQUM>D6(gQ5nnv|} zp(0t@*-Nl3! zWw4&FO}>1`4zYoo24xGc>96o~$!lSNht2dgW*Xc4*Ff|KcrN+ApVxej#wP>qZ0n+B zNsA`cj;or9Jq6YD1Ll<=5nD}zr$;K&{+3;pk+c)hnizWuDuIGcs+w**t56gW=~A#$ zw`_28A&bX$h{1it=qKf{GiAf5!;=XVZr#0oQ|UW)?35%)L&q4&5aJb*qIh<-uyYEP z2TkE0W-zw3wUZPEY~Ye+QDT%mVqTXQPb%rd9_SdJ;4^S^ti5*xLdaH%A@*$}dIBTg z)c3cd6;SRkvbe!afJHVA9_~qBfhsUlskv{}2BbKJ5qpIVTuvkt(MO^vk`xOy{0bXQ#MD`1!IXDQXe_1(%V(`qozW15QvOHQK@KbvT0niNQmtyx z(&z?>)#Q+t?Q;3>>aDm}X9rUUPae#RGpiH21MAO%Z|7Um(=-yPE$aTQ7FZmUL&!4o zA~S}^X_TjJ!=qczcJ39wq#F+pRevoXqsQwVUouv!e0i&r45g@7n04Y(A^YYnRlw!z z?ieItAadkhqu=#S;}Fvdh9(rgG0RD{)vU;IQ~8TFdd|DbB%N+)_q4-LXt3kQeR=^% zI2xSLlDzgnI5j=V&85u=Z0O9>K)5J+jZ)aCCHi4ciFr7V zt=yWa6ZuGYR6$^<6B;_Eq@L{utiW+?6QArZTg3G31mj5QzVv^7&Pq&`og?x=4?}B+FQTl4+!!J zHs&kD{>K;RZpHL#!qv{3V^%M!`p%akWq`Grg`cHDb3f{OW;EAs3rjghsj&_?lUI07 zk72YAhb2sgCB2&#RXxBN?A^!A%|vuPqHvKE&{Pgur^QeNV}FG5dhRPuM~Rb7nshxUREa-iElI6Q<<3fmwwdGQ=+BkyJFFNP%BC5g8fLI2 z@TRaqU)yX6#;eI~9rWJYOun{dveHlNw8rhyLC&I;;~_%~WhTj`5-NEC|ReE9qpZ_a@fjx?aU99w$@@h$=jp^ z4(uABq--&})ru1AL#Q`}UEI2sdfrRc4UA0SF%fAbHyZm~{+AG-HbGj`Z@^@fk85&9 z4l;vGOn%9GX3=MuFXgnYG1~lzxl1g-U>d_?yV~H+GyXO%$8At&t+9)VLpJoo)VJ3! z6)D=P8Yw3EaTP!nb_rxbbJXDh4hU|TeTk;%lWVs@AP(|*5}I`^+E<#upzP`nAUVQq zo+DDyA>%&^w!C!N_G#F7oG;d4YOur_cQ9U@yGkZ!o8s*=M@p@ie8ym6_MQ{cW1~KB z2uNZEu}-;6qIr9?ow@Kqt1lcJtr^iLk#4eE(z8IrNr_f;#*B)pT-90G6#}A!?PPQI zAhEPHtt1uqRcg9<^19VJQ-&tl6Z1FUT4a%cYj=VC)d;-S`2)r6xLE< zrmhJfq~fu>6^DtArh-%?MX&i>DCQFrs}K)*Bg*Qdf3qsvFtmxjx|;0f7-{Q;WvOFl zrkZ{uKH2kvOR)=W@Ol@g>s%9*HqIKL2NURR+EPjIi9bI#3Q48!TWBwmVfXl+u}neg zklDvYUtSa491mcd;z_P3QVsaAJ&tZ>+dVh!vmf!Ab^m(V`TXNM-z(^fJlY+Hh22KKyomRU z_DXQ9R9I@JvEW2phYZ{idC}q~-)}t}hpar9)ErWAbf}|0%{zMi@Fg<+{U#W@q=(Z$^&E2AFLi zZjwE>p)ei*>UJ;-bw$toEnD20v?L7mpgwS>F}3LokUhF806paUuPScqYhoLL&T6s?slAc)@4tddUN;jGcm$3YOERDK?1pu7ljAlv_9vACoLNL8tR5X24 z?4F7DG!?4S%4v3K*X|wVZvCR5PBgt#ll)redE}e6@Sxeh;-*D?JLkN<4@}F738z7) z?8Owg>vF80i_0?b;prFnB}DA5$L%6X72#?cJG$gtL5@A>_gfql!0S4S_jTLyZ1+% zXa3XxOtpmU@EGm=l2%_#B{)Z~dui0SaS7lKfu=hid$ZRQ)Mzb3uiY&nx;G!o>?a=< z@6!&k9$O(UuZ_fSWq9VOOL9Hkff9U9hFqb2KB(cX`^!?>U8oyaRemPkykO}Us}ZBb zPGDAEjwj9GK_zehu)IJrSxAx{){0O4EqeYUps=gSZHhLWi!h_1gk9QrmYifi><%%1cz_j(4tSbJv?25Y&>@k=;nxW zXerL2nI2OqF-6DYtrJ1Y;xc~~GhU9jO)KutAZ@>cOsj6Xuu2x;LV0`3b0SHNM~C2%depsD&gNgx za?KdFO``J{ipHTr4aWK~(m2VZ`Q`=yEHiLf+*wWTvmMM!2;*{t7DT@A1dpLgb9(C_i_^qFkFoHEnuBv6GreECQ zMEnbN8WS#crY0DFAt!QFKLrU&XpVWL?xNq88C5g^w4QT`c@nsk#=$SZi-w+?K8T%r zFTV&L8)Y?3XyybDAR8(*e-2nQk6QSYgw+Z^LfP&mknfrFynUNMA}_<=j~P6$u|cjuDT@vh#i#LeV?xPU0A zPHnUm%k!5%Iku&abF5r$3B7x~k<5f3jD@D3Lbm!Y-wI=vy)Pexok?ljaprm1k36a; zwm+*bge-*hAo-WLbX~3%Zqz!Zk5$Q7dv_bvg4TfhHk$ewx2;#s*NZ(3t>jsS%bv7K z%qbyG-ndWe02_A%4(PW?k8)mp^T0q+wOe2crzKj9Sf$n2vNIxV0?6L3By>sC#@ByO(wT>CzMTEoEvh=hg`4SP>nhpsst9#6_gaz!dIqHC`kc&Wymm zqdr#uUElGa3Fuj4Ta~q)q-^nu-&1cOLlN_$Tu*n;_4kQRx!{i$ET?Df9yLQ4Qr+K^ z;B&(xPbZG&8nTkG*?Zc$RWIo>O1}4rB(jZZuffYaNvGp>I|>Z*`ms-lL3q@XEi1gF2L4xQWI#2pX|$}(2bQb~I3S?ZN;57{mh5>MjHUW5np(O2_xS%$wD zuK8xnb!kADg}*)%6C##Y>y5vYMa#EGRHfs?t~Gr(C|!#fsgxJ-eDf|#3pRh90_D0i zYU#|Q;q7nSI>oQ zvSjbe0$~ZI@N9Jp-309u`|L-2yKBbQSdZvJ0ZT-05u2rPJ;~FsL_?Il;f1a}8W3=~ zX!k2n33al_mytriL{mIymNt5qbuDJA{;`?;C@F> zTScGlfy;2|cpJt67!bh3P_UMVe#aHd_9C=S>{4qqiIX|*^$Oe8nUOHRj3%6j_og;F{7Cu`ib1kzh?tHyf7)_SO z)BV0@U$YMEgP@1(izWrfE9Br%V{R=$s+vO?>OUW|bv4I1)89M&8Afo>KwkJ>X~btO z&Ck*XJP|(96{A=626iSWZ~>cDDL{OTP|Du(jOd`7LeY*JDBAkGIV(0pQyBU{mASdx zG+}*3{_u#S>*77Obgl7mEs!^3VlI7kAMx;YwsXzW)ENPew%<`)rN~pM+aj;$%R2ZM zb}nibHYwv8f>F(_(Yg$J=n=krv#+|41c-2~!(IEbi!+V1G$}U#>PZkxExq*)nWOg* zKAjoJnZeBZ#@yVV2Ryk%1F$=$hCh^b7`VfakQ9}|myO80X zj?R*bz3j+gBb=_SW@lVSzxp-{RTN5ev>6zn72>BCn2}FZ9mF(8s1UXpToJ{*?-rfG z`_^0DL^L?aVz89p+XlGQj~kpwBi*a&9=>`R$|?KmqYTB<8)`r`XdKQWZ}CvZ%(uJB zK*?~Y{lXycuraXuMP+Z^=Wvszuuc$BWy6Ike^`sPzDPLco9>sTj$aUlv3iCcN$n`P ztBFl+jS}ubBWQ@3W?GXVz-g)uBJX5qI%v%mYW!h*!#+PLK1 z%Abu}*7(eT-W}iBv1H7o^*pbJ$yX-5bq=!R7g-&mY42d|k60hxs0kU5D8|#>6Ubnf zvLkaKT73I~aZT?_no*jQY=!(aakR-NutP{~i>fi~3m4x!|4xPV0h;CNg&^9}_qAb1 z`xcU-{j$5agIk!vFVM<(`4yx><-om3ki&_uAFs-xV|>WQLtc~x z01iXL`=aqy2M`N)vg}*it~Cn!?PXSXY8f4 zC==T}*Y!Z+dC*Px{e!e=S)Odxp;8ZlshPLOsukR3K@AJid1CZ9w*k6Sw0Pf6A;R_X zXYP$3wQh7EBef{p4i+!NW8t?37o5@1`nqZ6VyNKyPXy`D{WQN6Vviz3cpP`PA6Gr< zwo@55W-kCC2KJ-{OMbD#-z)lY#M(AcadaHPjGh?qR#m1^v!(vBghnf)MIpx|zj|b` z6~Nb*pc*)qa|QXvn zb6hBwj!;E9e+k@_^4V1bA=t2c+ho7LKx)z3MM)VfC%V`iM4d4z8E3#iHXKZwdY7r7 zzMCIGQe8Bz`qz}*oQ35rBU?smI_udOfy5fz#6{()5v?b13sjz#)ivYTnC;->Ougmt zFjJ{<=mJsW+c2AP6G;u)Sl%zN=pKkid+4Ijd!uqxE^izINYFib6xK6kI23Kk^pTC_ zRYEPX?Rkkw+fqvcaH9ZlKxI?~Y8gD?dyK)0es1>Y&F}_K z#@#Tn9q(sThT;2{Y3#fqXlkjs5_9dl8t^K?yu$+{{TzT|?0sxHl7EKL+kI>_KIPFJ zUmt_+_zpf!fO}8X`G}%~+=0 zRXI;5IS;4u)%QPj#1QU=Tz1dIXNM@JGH+g#6~PW|919>nQSKn?ZRoR>(oXVG zOXzta#@Qf0qhf8cV|zsD(V`UlapokxD*ZEDJW!5M(IFuai!hKS1yDf0OTcVYv9W0I zR6p-rbFtwdPShfSA6+F(w2Y|e8wK|Kc=Ya*^+GY?9bTHe`gt4l4!Yu;@Byosf_NL< z_|IB#5Nj`yJa}~yv9i0$f)TN{`@`rU>Z+zIOm$Ef{HftSGXa_Y5O)s@gZ_4YE+o2_ z)v2<;Z2*feII?t&2`jOB+vfyF+#UHQL#)POhvb0V#@r?+v~=A!F>wSWl#M4{*CH2t z0+Qx!Ll79fG`VT7o)~AdPYdN3gW6&AVnrbx-+T@F=;c~ZMIG528P4+~eT2FG(d@M) zJp*@ty#}o~b6c7h!!!W{zB>*ehT5J4PoLaxuVUD^k&qx!&LhXqrl4^D%NkGx%U8TP zUJp~8Rib{#&unoWlHnQ-xK1kuT$8v{ZvKmu1*l8RY&Kor^$9_usHRvIn z>Ad?6qKA~6MyRdhaA$oF8)bjr=Aeo8TAk$nNO8j-M$m@8ZL{lCt`0FbYO=XBXZeNI z?_S9=9N2y%)p`-XT1^jg!Fl<$bpG{}ZzDP{bMjEIRmZqJ{JoNa)P`8ilT^^5 zY^Sh>pDwEYGRC~LEZ5J{=~2D1)A$9VV%nx7VKP_H+}6jVwd;@i`snJLv?JPFW2+Cd z*IJEy{%-Kk4?SIJ{5RmMW@_PDI0|CtMe4X(9BZ*g6=&#wW+4G35} zHv?qcWMW`PF%|#(4s0J72!Owr^_3SkHvp?D{w-g~i>wy_XjnN3gf+0dXT&vtXbEDg z>zz}8x_ZYQVA6HA&!_{~*CYVHuC6M;7zRKCaN7n)TgDoH7EQk?fU75&m0vg+Kg_|w z)dTYS3*W)mMmi&~u#Gin->yzlgKGl-usMy6Hujsk`c!V%yY0Z&lgR546>JMI*Z1*#(bM{mtB|_8 zy8kb@n3K`uSl->L#9-KY^{5}}# zpV#-ruq=FB0G^f7b^yStUcO&k4DNH*#@5DrFIF{>^#DNXg9OG4+a-)2D~vBIVp7vv zz&k_3QxKrEwY4?CE;(@SE^dI_->6cUfUo(%k1HxGv%m0fVy)?wgM1s?Ke~bd8+=k> zaCf==$k#gfC2~C9z3NcEyNBRsjX>);z!!}U4H>-NfWNVWzjP+be!pZ1IR`(#>Y>z% zHLR?@(@`(E27o?t(D;`?4gP6Au1LlFEq}0Pf17@7es*0n_j(sA0%){t_+@{!(3^ib zmV1@LH#GibtjTd5*h1K+1i1N4ClSz=qyRX^MKitG))<(9xit8%`6pjz6ZCez91To- z=&df%5R|OA=$n}VobIOfxtRWv2rSGj?Z6nC0qP$f0r_QR@#lt64TiiYhwfJ)laKXntm^8ItlbnqQG^doZh%Pz@V=!;#PgD|=q z7Wj-||1DrW8}rL8#)_|F0ta}`fVdsgW1qcHNa z-TH>_`@kcI==DE(CXZh<0a6zlJn*l0Pg|b&m%BMzRlkY#zXr1;fWDm@0&9N0S3gF- z=^dOLJUsqC9pQX7Grlv^1ahwc^lQM@5!`G>wi9^PDp*@SDm0QuGfnTwT6S(?>9@f1 z3uhmmmrr7Q(*>bl+Nw@n0M7g!RT7=*fAnvj={GX*LZ;q<9P7QR>=WHtzF(vnP{;qW z_1-}6l-nPgdv_%eTVAYIUvukLPaZU6-VOA!u#WDAsw`$b_?jF|m(X~u=OOv^W<|G? zD;6eUigtZ+CQ4=?y##c9o$d20hn9wfF`M~&BYT!35$!);5539((+MhK^NRyj*Nv`6 z>G8Lz4?m9?e4lEb4akS+asbn%ruL2!jMKYgBhc#WG+JSVb zU_<+ylfx2M{_Q-RU+vaJl*p`xebuCszf83`kn1zv*mNyUQK@pgrL=6+{HZFlVGql- zpo|pMI2VmGC^d2|iY|wyn@#i0K>JGpjdQpT^R& z@gaeaP__qP)qyV70SZ*bn@3eRXK^hNCJgpAV z%$M5DG4C3p^co<$`r!}Th|9g(nn)(Nqs0w%%6ycS zKAu9$zKb~Zl3nWoK3Xkhf&p|o$((rcO}x!I&3DDq5P_x%=lAUl_c7lyEK_IO0 z1weNwPer4~)Ytk1i~H4K1iMlQa@~TM38wsLp6RG(Soo!c9|SJ9O5r2vtm=!%5FC)o zfuvDrRU1D5V3`7oh`Yr5H{ux|GZQRlfJv?}e4?{XSM2io#QKUgch4G7Cb?epa;!_u z8mV}TOC7l(rv_gjZq5XL3B*Dc{iz%@;MhrM>XvDjnHdO11b)7M7}%_9YaWVO`|D0) zk5v1#H5JBpx=eIXq@UwMhBOtjKzDv>Py=AHf22)HW*{F#L1->J%*&t!tu(Cb@JfAG z;nN_LJ_MM3LN#lGbp&{&{OcM*uN~1iHO@>(u3I=OUJhSVM(kMJ$xKo82;s|&iKv4(B!5OFR%%JW3nTv4yE01~@!qs6T^N{Ln1;9Idq<3ep1#~KDs{4ZqCY2VBS)it# zGg~YgKT9ex8#Nw*B{gTrAORQ=eBDCF=*^le-(@c({r+pioll?g!POo5EnVbBu#0&a zzi3x887|VuZTHf(wk!&dt#o(kof1Wg7$0ejp6uG5yK)va7Yx>};#TrpZUxiW^E^px z^&!<=Pa8t?*d-a}eNzjCL_o||!+dT7QA@j9;JS zE>E(n$8@L7G7=Y-B0ko>KG)?l3MjpjL8g5pCn@xFFkB-|K{p@L!TC_G-TC-EckVS9c96U4#4mz2Ej!c2 z)E?a#-=WWP#sL82^Hn5Xx>anIa7hXxzs!k-?{Q?W}jhxI5C&{g$-(r%F zh*g+mkbCjoe#{`y^06Jr#0RE=JIV!lQ=ixEIy&(wwk%$PFkeyUAW+7d=B>X5lobRlznQB<#3D(sZO2jDd;`d@E33D%4cw-9e9sa zjaf9v_afs=-<(fU=iDwug2ar7NQc<2&T%rQ4(Y#DE&KJhePTM02jF?0Zqt`37WH2i zd4!Zn%zqQ3DJ%%hwcLB6dM)~+mdP>R=+n+d7iu}KnTT88TbwJT90Mijm`^c~;m>5Q9pvqclYqDk3{!%n)CpmRyWlxK@ z4#Ic?;*1x`#)gcwUU`1MZ&-1_GZkZ5g-)!qePQph08)7=nwck35^(@qhi4Q5n_pr_ ztR%pPxv)n~Mb*6&j8=lGjv~|Amp@kaeKY>W5j0YpiLJRxC(8EYWhugi_&($HAdWH} zg}w@@0Syu75#1*W5|Jk+1=RSW7E$HY`aC zYoE;TjWw{iMk}=~ry(|0^MP(ge5fQ-72AZcwX$w&F)|qzYG1v{FN;|qHc(}gFVO-> zC$j%k>48VPc!tWd&)SG%yzY?O*l>|GD&&8wp!+8D@w;xq&Zw$fTATX8Ts_ae2p53Mx=zZK6j*a%L{$O zE4`w_+^fa^z8-=4O|`GuIY@?-Xm1=rkM&HOPOK?+hH`<`u5xBHMDy(+eI%nKQ+%UR zVVg*sG^M(ECjbF4?MCMv`zM?Tw*UBUAD>o#6J5K4dDmdjl{XyMxlefT2#YlP&0-Nx zi-<9*_W_i&zfmy$9}iH^gJ?vB4j6yTFpep%Ott*hrYC~YO1VK%egG7wY@$nWN8(G1 zim=Gi`%6d@__?Tjms!<%_xh00e0X>?D^&91XJclp5dj3@RtrY#iM}js`lHZPV;!9(Ib=Z6ZYu>m4XHTIbjOdIEzo9pe-hCQQ3vnt@IH2Bk3R7)=siI z9?Wc14A&@~#VLoFJ7;;Sy-8E{4af_oW|?caTH?!nM6LWmZ3bRGqY>)a*uEX?d)@%H zn6H2&r@bBe_wQ!r?H77c`haam3Xq~F*4C)(y-&D}$@sq&j@D_!_2DYI;suHbU>Nw< zzkVJ)v|Ds!!S4aS5^}#<+R+gH4Xg|KW+0w^<5L1$@ z_)k}&E8#qO@sNdlC>C7K{I5N6*#Nyg8F?|K+Lj??h8#GTTa-#|optnuv_G6yUE5cb zAU>;-363+|CX@7K(775dFmdON`yOvB!t%F2MLC4Igngz~9P!ZLtxbJJywmUCoG5~) zt?79RAVj{oJ0;=`wS9|m%Li%1Sh3DWKq3BD2CP7KzTdrZ6yrrfyfq@O~&~$#&+m8F^{1xLE-R~OA1J4~-%>uV znLPMBbF~lq$kmN<<~|l%aRw#`IHtB(JNHnP3vH`T3os2JJ{zI?+IsTcs2ALt)02$z z^)hovbx10MgR*&9Lvwj8ES%q$MQmT-vC8skLE3E;fgPvbzRgLbM>Yvzx{M^Ke~~t8wBF2Qw=$*I2#K1}3>?q*KT!GC!9B`QngD zr)2$EpyX_`9U^P{yYdzgf8oj#zk*aA4)ynMfUm}=nU4(x)!Nl+Uh>3@#S8wltvLZ6Jh(#ib(geX71+Z)7dlK^d+VMEIdidxU0|Z3t2i#W<6Ej(fG`ww+>WM55ka-s1m6m-X<| z{|kXOEEDOfHHlE4xfq68+HmD7GcG|hpy_J$`u3vq?&qctYooyR;Dgo;dZ@@5L z+LowHPSoJoyq~FHi6f`kK|bgd!jR7-kt;SEuQ8{8+nWo;4I`20Rt_bM>}sHaqSp6P zGKY!r2H#GdPVq98GQyDIUIE3t_s%Dj4~YD3vDnAFnybG=mVYXKZ}hR0ANTVN!(J3H z2|SY6(bU?0+w*b@M*8uhV<7+$6+_GIs8Rk7eqN6(2iFfi*omzLU4%Td73KM|hTy(j zQ%_LwneW(fiEXWA$i;3X6EhRov=+EXPUPUD2+# z$9$hOY`jB!jR-ll90UfD_i=JR=Mz?xY2kvD?z-Id>RqBqj4$#s45ZxL=+g>3UF;32f z?QPDYO&)W3Rw#Zb3UL#GRBvxwZzpt&Ne;wPv}^jC#B{Y`+F4m4O!wF5{6Usyr3K=Q z##?1{v+<`m+-Ax}^QwdnX9%FU=sVlG6JlDUhn*ruW?bDZ=mlQ{>r)CfzNzZ*yJnx- zTZHXL$H)~YYbv@BtDp{Vv%LoOnG|X?U)BG{x%5GSmEwSZgL8mDJ#oS1Yiy0|Q);_J z;6aMKe$fS#LezX_)$~0k=}=oRt@ys|Am?}j;!Ur1Pxelhs5i3a{DgG9kRZkiI(W#^ z@LIv@u4j`r@(56cL}=8$Oq(G>VE*5ZgFlsuyZNlQYhG_84#%)h`n)zTaesHhrQ5SV z_i62&=fA^#etq`A;Jv!V)m_v^UfbGR+bwnoyOOC6H+{+1!++| zWOulZZ~WGMCX2|!gDj&1-I%B~2YQZ1CP4Aq7wwhTznA${l9K9@yjLi`&fnOFR2QPs zcNAcFlA$_HU2o6#Rkwe4tZxUzs!h%*3Mrh1h+$%~(#^rbb}lx<+G1;~;H^`hdn-${ zd?v_Vb%JBot<*%NE|jDP0vjuIP&qM1B%kM}p5Rc=dI)W9nKm1^v@y%o@glzBvTZcY z?Hoh`;)xbQ#J_Ov6Q@M}=`Y_+x`aDu-Sw@6vWPGEj}|b{fdh#r`E@ z)lfV{CHj)$bjE%_`_R#o9P@}-P3e%GB18*AMW#LQD8?qg z+5c(DyiO}vP+a3aDOr53_=Z4-oI&sxt)qv_$~@-EIIOb46zkm*E1?Tl^_g59 zhD~!q7y&1#DaE&eZtcK0kgRUlqV6T z)Wp5sVxCfG+YxRT;FG72UeK_}igiY}OTO?pSmH-!l0siFj60by9!S9JN^(;uKl{3V zD6PO1jkGL^s?|`bNI}e_!YF?L={SvzvLMW3@cUlg!e*+7%fCOMOuYI%Gta&OSe@DS z>P)HImLb*3!h>Gw*smTya6YSOyR~k!^`R7@qw&ox&aS@r1Gxrnbm{h9og_`ppS<_zLJnAsZi##1R30azU&O*vNmhJ`Zz0w(Aoo zx(3{@%fmHQqrX*%P2~)`G%V5`>n{ecmj43~kK(nHMAC^+&flQp=7uV?6lyrZXf@Aq zkuLkeoB2M9aXV7>W%sh?Ne5R74?5-(+_UU#c8&y9>MFb}aTUFM&e)r}d(A@d_u`IO zk}ESK49Jne9nO-sCMGoAlZ>Y^DvgDD`B8(62%MbX3IrP{+2($ocFi_b)!uZw`)9<$ z5oE;^`Yq4a2z}7xyyxruB`$CL9y#<{>8!($a3{7P;Ao!1LMM-6HazC;=<>mcv1_Pe z7?rl2ukJ^U``_RIBaFc zFWFBS;Vd`MuH{PGXLG>dK&C+JfZg@{AtF0_vXS}k*Lk`)4kA*717y1hco@>bS`TJu z89NJA-xJF2=}Q>@flFn72P~38AyLsJ*P$8vvC#4`ah6LNOB^-xsd{u#RzqE?65{=o zk{7$u<>;~P{c@Y4u0Vvwg-sNrx?TtxICk{KgC$tm+bd;c^E~puJzS|1(oOQncBTm^ zkZ2vcIfb&r8!V9zSy1`JWWbmNpB|UZhIm;Kgv^b$Q{ADOY6T(ckcqRe>;@3QUmW$y~(!AUsZXHwme!1$=)~X2A6*Rlu*>jlHCv58l_pn za+Qnf0@Oz^S9?7J9KNa)?a1~b22ds;%srE9GcuE=kdcSIAgz|1g@%0!rK*m z)WLJ#nuwz2Jnu@^3b#g`z5AIN=jst*O-JY&DCyRNE9IOk6-OJ5Cd+hX?cJ699NS6u zWtq94-4v>uTE6mTetZQzY#t&&4otvUiQa!*#{How#=;Jwm@Ps;wRYt+ixqB6Xo^Ej zN@Os#o=~8~D>E2eChF_^(4_4yH5OJ|5t8VIhs*Gw#LoNDCe6A%t->_gBAHIJIWQ4K z6)f*B=Y5$0(J|)EeT>j8$-)ThCeU6E#H)p;92OwsRscd;VQ@v<-|5;MS_oq-pE9&;4_t<(yA0<54S| zmshlsRx7;py$+HX$$z{MJ z`ab5MT$w5Bj_Y5~Bh!EjL}i_nTD0i}2FfQqYs32@b1L?F6PG~8O`TRS8tJ@Ar_+r{ zH5-3akw0eE{K^+?D{KSiM&loq*bJh@!j?~A9vn%qo4P~)TqCaO8k0R78aqc_evUI1 zW&BJOhPmH9_|_&CRcNMU`85^X)4tNy&B=2bdxx#u# zUaDp-M#hTYB*E}a&Mr6cZURc>%_~A>S)-CMcN!a*5|%ug?w|nm(y%}WG}HjOLJC~> zB8{*vh;xiLIT4#(08GMCL*?^>x%}d~KZ6qlqqqR8(rLz^0e9m3vsMX3k*Iasie}XJ zbw$}~BDJ;;py#`PTqyIw_;|Imx(|1LGrS|@io~=*^1W31+RArP+STQve{!YQ6VU16 z*o&FYH1b)jy{>sjs{_#4qmly?9+3k!Z;NXor$@#XG!s%LlI**sM`K?$zC)!mlJ0?T zCt(oFN@KU^9@x~u-9#US;ym0K5-A(DB`R5g;5nW}yNO4qy1BOZ;r$@3xkff4-!^&w z1he_6D)u{VXjW_S*ft4H7tH^gp`Rq^kN`;94wfj4IW96ZDR(s5FgRgG93~3ctG$s% z@?cIS%1jD1?N>-0v}si>m6a{LgwBnEX=Wx51a2peq8=Q7_O6>ZFaTq$g1M zfiQ~QA}cQ*B|g=Y6WGr7dfq5;!F6wHU}C-6oIP7RzsqMSz#Al}Up=;i>K@%rwvJXE z)Lr_hgR^RVsiLpUa}Xd$sD*&X*PvU7A|9CA^hIFT-WxDGTflri*Wd=C1NDFq7#3ba zjt?8&8Sp{TLZmf0m~=R&{m=r7F1!NrbO9;NTas49rs%6iOHEx@n;pB;(_zHQ0au(ARLFf43?B=IlZnO}d7Me(aK22q` zu_|EA^-S*Q{*v>W5{?!m-_-Q=K-OL;Jpevy;_D!F)SP>RA z`)IId@-FdrqL;`aAsnP+GcIl)k8%g!@I|T1X(d0PwxVj33Gm-KP6gks7{Fx#=JOL( zHkJC|4rm6bZink1F4UG?))fKQa|fPm-#_NS16}@U$@qac9n{H8TqjrzylGCFI-X3^ zQE@h;dwDLfG_?%u{yIKLmLF*&}V`IBitaX!hdnnRz@>6N7t5);7W>tv|vTO z3i56Yo5Vs_ca@CUtOMS_yWjSvwuY+o?cOx=nYxAyW%Pl*)w5FxMCWoRv2)rh(h;KR z@$K#25k5CLPN<>~ViS6M1`3H$dK>JU>t?aRcG=m^-V#l2HF1gUs~#CNjK<3YSTb9lviE5x9B&>?lCSe9^n4#p}RcP4jNjXqSg}il(0O7mG~NSEF^!D?re0H%+c-QQDa&f z4veQMcxQJR=t998&nD$@{A}u=LvD;$mn8uU4$Q}@J;4w zd6bOo_BmBGwzkaIFUChgTgF24cRpmz-*uhI51!RItJKRr5c_i~B!2~d-RPVPiuB%6 z4(e#98&v0DVg0yF>20o8kSYM=THnup=jOr9cm z{SXyEy2%(PQMoo8JGYdhPK4&#jfsj%^A=qDyO<}7?cM*@ikJ)~U<>#pmBJfR_7-rD zAVoLA|Aqif91q?8_Z#pQ2U@)eKL$tOfLq@x#ST~MHVwLyGMe#N3}h7-w_#0zvPw8> zr3UBrdvqEtx?iDA&ARYWIjuVMhGER>=zz*VKu0I!F;80?)kIOJhuO@W7 zwD->80Cc(EUuunJchj*?|7y*)FqNp~d-SiiwWROmx6uP)Tq!L2Q8)DGu6eKX-7$c?i>Q3=_7!cLvJj%0MyR{sbWBXRR@G+SB< z9?dwkl}q)vug$G+eMqb;C#Y^vf3fLud%`iUn%;U8D5-?Cn?h?n1U>&D)W< zsyKT*X@7{2`2ZJ0`x+G1RlN7l*qLGXMQ=+8)OPr zTSujgoh$FQH!;ZAI1puT`A|arPSdk%p$RC$fZtH8Dgibmxm9cb`gv35waqTL`%09; z1d%g=$$QisPo5>cT%QbL>+S4l;VrJ&0KxD*@x%h3poK{sWSDo7D)d^PNCfMXar*5H zN;79MPB~d!m3*}Tvj@*YG9%qV470d7r)cg&%joKfUVN8_ilfs!i+kcVmaXK6KtZWu zutqag#wM=^jt<2y=qq1o}X z=jP&~5rc_qh%#CEK^f94oE^<$A^ERqbWbIpaxlBIPu_G09`VlMliz_L8YsTFfmG!I znXf{{ry+WhJhI_%bX{9q(LDY3xH##=3?qtWl3RJDs}Jxo)sOr2U7q2ti-GywZ#W?! zl|UO-+VKIZm%}@)=2(CxcT^7%wd67IT%H;a<}|Gqn*uOYfhh1!mwXX(RBRfkS_>{I zS5KQ}HQq=41DJ=6(qzYLhSHxlqG-W=i;D_$z2E7bNLy+T=&od28_KL9RPAhoM*Vv1 zEzb>VK+eUH-C?iXo1PoS&7O3wMHcvpI5J?_DsL(wHqv)QL?n>PWRTvEE$d>!NaE+0 zG40+eudnKESSd8&%9cD}q$rF$&sY_TSaZ4u4Jh0+^6a`mu78PMUG}`XKIS(44^n|j zZ|mn05>aD^*bw%0hBqv4!*TGsQUgEa1L{aROt64jKG9kr>KM^|=f<1BP(LOTXATRr z)=~*Wx9|*!SQ!)D+_UNXg^~U!5cu-t%`D?zx0!WA4X_qe64vb8Z|ca}nG zzpF+!&h~s9I@8IB-lnF5S-KiT1Q)11^6sase4%8(Jvm&9J4J}l3G*M z#s#?kQJxq1Giwkkg0|;K=I1QL0BoP={Codz(R@K z0nK9cr7O&`O(mt{xPC$A^xKuitR`tV$8xYB(NM-1Z>M(xRiJu#OBdi!4sAXi?kE=u zSKg9~hG1ZZC}n`_aH7DuaRF;=&R%~{`JhzRM!EoBc-jKt5Q+AlMwFjH3{|A)ihN)% zyj1RtPHB?v%s5_JT?cc#Cs>afmds^FxC^IE;R(A1OMcX&iH}ljj7^f&s3af^p2u1> zb4By$b%h{uaf{N6*3+%Gne-7XOOZc*LF#lg^?SBP)IapZP{%fw;wdO!xGhI(UO>d@jE^AI_%Qu`|t{W?T zxSf>ca_ur31I)!XOI&s+HGTnYTn4O160I<7(wzq+{w*zGH%jWp5atML~PUu5pF~#xlFOI-` zU<8ShoxiOkLAA8lBY=EGe&Cs+`c-8hV&d?lf&dEqDrx~u@c{lpB-saP2L2En133V2 z07S9U(&mD$uUG#Fs{Zm@&!}4eiG$`^+|60|BPAgQ4*q9o7$X`Oq7Fo>4PZ)OMIQq5 zlTTSQCszOf&W~j5g3ri=35|RWSRcX|_{Rk7=ka@W{H@kYN-V7V)Hk&-|Q&h-FL3ArRkn8B;e1hiF;Tw(cW$V zjSkBh6-^J3-{QU*Mt%6WV$8MS0Wf_N(_5I2*)G;0 zM3821A%4r4nLS@tsJ;n6kRLkfNib^za6k?&V7DJ6AOY&%Q~-;x;6f5~L)g8eeb6+4 zeA68BzFwu1qkTX(M}R)xbZ!9r*xv5-b^I7w-&z3P&W<;rA6obgKGjDDfHeWx8q=5G zqWK^2=3px157*QmBTC=TAFdXlUvF%PBftiJEnT~&KT!46@aJu{C}fUi+qoa?QmwU_ zBSRP6FxO7rL&CDA72_})K|2v-u`dR-*OE*-Vb$_IZn;e?grUB2%C^1_?yS+PY3sIP zS|0-J%sOs_JM`lIk~0+~E_;s$M}ZKM-#E-DY34%y!e;%^y)pF2rYHB zKJtT=VL#eGi7cXA8{~DyF|@PXRyk>yV*2I6sdJk8|41TttyR#{;qci|UBvLQ;pjafyBWzvJ_UiwLL`#gc5b5>X!y8Dg2 z2PgU?!9yD;=ocIOl5n_mqKQ zukoK2T#!Q@#_Bz%S0JePuN>@HUzPALjTcIPPNiv$k_Zl`3ctTlH@K;c4!0bM&y6lZ zow>91bad>FHHU{Vq%?_)@s!y}@#&?M2Ov-Fc2xP#mJ6H0rCka+IrLOrF3ZlAE0v7n zW)toy>wk7YS?fhbz%Cr(>Qvivx1*UM0(W(aw_7C}{P+hmxU4^q$EZFL5$0Px=D1~x z0SruhCuL8Kaz*w}UqR17{V91$D+;gVN=ADLiBW$6dgVFV##1Bg%f8r6EF|zdI9j2Y z#aCsCDFj_)IB_~J^>d*}&0&e`3rSe}@`{S=y*0dIil~TXVCP5A6$QEO2g9yIpbIvW zMHZn%aplMpoBQQk&BL(+xAYU~z_#M1*q}|H?VLjOq6@4Vx+xxa5S!jje>b=j^?!KH zL&F#3EPpWMdZg+hV!v3zm@Dm%{za%9%Poeq4AD-NNo}u$a(=$m}gIt_lNb=16q?kwy7*a}X2W`q_D z&`ch2+O)p^+<-=@a&6WnsI!+ZcB{Ge;|JwxONjp1uCpzqy4{lR>Xif`tYw3BYh7;3w*|+@kbIc(W@u?eUDZl3 zXe#(V_iey5LJX-4ckZQZk%_ul9o*o_Q!A?Pf<#LjI9<#4!22#Vj};~KRg8*e^d%-7tw;cGrg6Be zav6{x;+w|R6cmI0cQf5biVdeXr`zL)@?vbip9#Xm}VnBAOJcWrg!jUB-GF={X)}=R9voq(kfco2J0C{@NTo8H@_{vkB zTOTTw&-Yn$0Di)l{L}>)*v2}wF0MjQJPxL5zX<(ouM3Sap z&$%j;kg-%hhk|mO$}6=sYpGMFc>1W-3kWf^0o@ct_ad#Lcn^N~;>kBnLJp&JmOQpG z$rh&V^Z_d>?TN{y7D^fZE|L~Wmo4P)AR{<#Uy(~eS5U7nl}tVU^WZw?mbfVZB>_;! zi*@hSb&0oCuA7{rj##uHPua)`6q7A^5gWyW={K$*9lfJ8MENeu@*QZ7&xa^1Zz_TE zba9<*K?z+8`eYXJPWaFlW*&85@nnPoP>gM|VDOnI#a4(oVx4MpqX0C2VkG}X5gGvy z(Pz&d*3VQ~7Dps<>v!yv_!MC}V<8U+U_q;|pf3@@<%JjFs=KckVrHh1k7a0`y;qKB@*cw-=;N{Z_g2o{gkNB}Zq+Ue+ZRA^f)D?w zu_Rq;Bki3i6@GK}-b{H`3{ap=p0i(HdmRvcp>}N$VH4{-mG=UgGIZ z)Mb^F_+0#~K28vd?6kzOR2Vs3`MLH+sH0A@zD6kW|Y)Htz z8<#8)e1E|<`2u;0UrxSIqTO%z7{%<_Ypao(;ptNitL1hSvdDFQO~bklgOba+YEuH& z`bcQcDgnYz@sV+Eeo<0xMW%*50}M^&KR_m0w0hRb{c4L%wTb3h?{02EOwYwISMIgA z5Y4NC=1sz4Vwm|Rq!L%kmFO25?^znhvUqM>+;ps*LbG}s@+i|UBWu8EoRk0F=Ngh^ z6xzx+n|)1?$5M?+JCEwbQXR>eF>hlQ@qQ8^9XZZW*1-W`cgEAOi<&U4G)AREM5LP7aIn1cnHlvUNrYs4Qso9zf&Fs*M& zqqqX#&rO=3N25U0QIlBs$yn(je$n!601-a6o!X%GTcWst;3;1?{#4 zO-`NqGO(7)qo#xtVkOIuo(3tTGtLGRbV%(TMj@x)W>(9A=$F>wE8g0slI$zj-#Cmi zc`Bth`L^NXWoJ>PA&^oOIRNW1NdI$%ve8%H85Htj!opJyQqv^pmth*#D`GPzOorwU z-__C98X4?ySivg@6Q*s57p%y2HDAhV>1x6bGc$q;ggCKm6r}d!uBuM^`H`mHoZla8+TzKuR!%KZ%Xo!`z+`16RIai+dhK2dxK(;`Lc|7d=zt5;t@)3 znacH~73pl)XbK9Br$X=s%dxr!L(|(zjFDd`8{~wW5a5w3lz#mAV?2C^qeXHoCP)q% zYQm~%Xds0xTbtk6LI!8K!NscFB+3;GMLCZ1se1Sn_1!s&&2J^a(N>d-JliG54Qe)C zo>F!N)|s}DTnOHTgu}gLHvFJb%3sEIM2lRzz_nc!jLzCJK-O1~hwwe|*=-uJAqE9? zI6u{yty6rZ6rJ(xliULlk06+r)mkuK%K(o>#X`?|ASmMMb`JC&RfKtcc@{kZZ_{kW zR*zz@A;4WhbZ$_8=#?A9Wi?Nw?70xFBDQ5qI+&uDlHOu3@;TO?D4g1nGpx?xhU+}m z;h835kw@K0u2c^?awA7!TDp+1TmE+D^U5N)QDq+gUYHfPlDxn)q4%nciYBXKe#i-@ z!CLwrj!yVh9`Q_Tx-itVGEeimI&dYJ1HNdo5q-=bB~*}n5O5IbD?0S6910%JXzqOg zNdiNfhGCgFAQt_8yv7uINxtFG-w3t6_qnI?8*x`r=-KWvSCdmU8Zr@vOW$!l!!rH( z=&Z2QC7#;RdiQ7MLY@&Ang5z8zrx5gSQj%>@337Bb{v@Wt!@#l;;=-8RU;3wh4ojqtISPM25@vnQH}x z(i^8PeSXtIrw30DpHr|PN=eUCMA@Aqy}fw7<0WH7(45DBWVt{>;hXo^oxdVC{EjLD zv;-09CfPyhble9E;3oO3=zEI#0zpi=965^sKVW=da~Nfc`U`&j?fhL#RLO{ zRG4dJ{oR#G=vZH~HX&S7&n^&&IXGUi4g~BD7ay2+)fhI@#*2l_{x|CrWuz&*SFkgQDgdq;TEPFFrI9$UimH8tO^` zzxv0taEApG`MTl^{RYMLOO-|z4@JLRo-ImIdK??c0Uhdcjzy{@iKv|74!p6yQc zYJZN0dxvRZ3q!>(>PF2s1a64u0Ux#Wf!~fl5~})fl(W5N!TttGd-me(w>x&DKMC*| zM^Lt@f0%Pu7%yAaj%gniXSI=GZWcL(NCk46XFN^;Z^TBnFT0sl_p4`0x8JF=O)xWb zl$|C}Pg5}{VHn)9m!*enhtk*tWhyUK+fmO5Y4~oWAW8P2X*_ftviF!vexhR(nRXEm zcr>RBUH1wsL;$+AMV=odUkXBF9kQQrx^KhYvi;oHe-0Iea~Fo#e!NXxDwp3+g1obP z;1i@uA82wPL|0$^{vDA!mb7ZTiO4vUz;e$x6w-d(T(TiS@MDOy?tsra{ytEWxNG2H zX3hyemtMw^py$uk2zFqULE8{S88j2GoYd68=|{-VGLN=KWcOpc%w-@LXq>YoR7~jt z{;bWx^@z(1EL@p|48Db%IRVKLI68%iVd%ZMSrKasQ%z<3b4*Ek{%4=i9?IYW2140E z6yr54xTGJP*03DUQ@uQ61BbrS2x*xQSC18$RvgoSbJS_y9IFVdBZlRsaB8Pcjce+# zE7qRb`!BM6`n$*kl93ii5H~Syjdf|naqrsto*uRSbxg$c;1Ob%x3bWm_xIt#V5}D? z*-7@HOw&I)P8vTNE2VY`n=As^lXEb4CB|nrgyLZeGwTf#ry{u16S0jmQ=rsCB>B1{ zI&i;EQUCaQqb8Hi`k>Z*8p}#@F5B4`TbvoP+eISUz zcuDg#PbDX4ez@609cF8#pnUy2W2z@_*Z3R7tyPyemS z)URRNwxGz_1^Q13gcxDH*t4UjYVSd5>6ouHf^xf2I9x=XS+(WxBe}u=-ZqG(kVc4R zJ`n8?+6hB3V@XeCG}O+F7_}xw(?%6W&^urHCt5~Ts^ZciU$4J|@%uuOlBjqWv{Lr! z5uK<9_Km!DaX_Sn zkh_meFXMVYb#_TilP$s|zU^8gJoxn6#9a-+3|-8QFMg9{Qzamw!+>B0p@NV7DPfr| z(>5F{4v(fb0^lxto6#uB$eh;_=m%LjbDjpN)+`p0wbFz z8qBzNI^2ce$Lkh$sUr9S;c?A7wrgV_3nuoOdChO(-N?O-u>ju2<_us%MEH&T!V}QZ zDEaXmAovsIvHelu8Gqc~2Eki>PVeV_F@Ggt{e|qHu+VmcPGzZ3m|hVEQEXGMit}0_ z@BTgPG($n?AGPeb1L7muzvQgY3&}y6Q`BEIc4LU4lVjPJ>bS5ZhZU5g6qFrKn5}SZ zSqg|Y*6UoP#KoUiRb5Y>?~SOuay#7ATc z6IRQ$=8`IZJPGTfTd5oR_znj*GBYR*B2kjk8s&u)s#*=%H z_Tr8h6874m8Q=xs=tnAN!Ll(3Ux>G6N4rJ5v#fr>`v_Ez{p(k!7Jh()XrR!3)a%Jl zT~eozW|`>|V*Frp1&Psq$+SvI6Ey76P@c@e=Y=U~>{iL)_hsD9Ogh*&$z}!Gta`oWM<=0R6u!0i)?!v`S%iEfqQ+lfUYWB=CB{<_f`80k5d|aJ2hI|6?O|+` z1CR)|4xNpU?@dj`lh85CJ{Ub%sFT*0AA@CU`zjUf_-jkdZ6B-SZ=lAMS-V+j;R*gs z=}0Q~jv^;$)dpEJ23_NLtjvcmt!jfId|om+(Z!x^M?4nbhDe7jT()hlEOQ3kXM++T zRT)_gerDGRSdDYqQ;Fp3nzH41#_Y+y4RA3*(o>9v!6bR4Bbc%19o82mIo-w^Y@S-Y z-AkmxDjXaAo$T=;%n{Myc&hH~K8M-C7&l4&r6-TJpgyC3xOR)L`GDvkQ5fh&RhP1PZE@}{&>Tesf(me4?` zosRJ^+j;Y4WQ6QMHWIZ1uP@)Xhyjng%e=*|qp#o#up!4r%N9V^ZEC8nwN9HT%6~>4(uc z8mKfqEZw3>KYb7kztQeeH>D+WuGwq$!nrTEtw30@-%(VU9jhLQ#*{A~-9fOf-&Yo* z;_3p~{g@s~dhFSD*vmy1o!FhDbUgUt@R$prb%n1VD$KfV|w`QkR+GH zLYijr$W;k{6@^eCI6#qpBGvs^TbEX&CroXQfEC%jY~g(0)JX$0C1D(}IK*J`V@9v)11lub)!?3$y0D9FB`RY zxx?7bwP2oAQ&DR_;nRKVxhk2f@|cDOg<=OyKvo67`wR5tB_>Fbb-8?sZXR}eUQKQp zZL#@!7G9tWX_H2X52YU^xo?U%+^V84v?BgXK~-sF{!6KMG@{!>ynma}gYbwG-yu7lTdEV1 zb#wSA#NU1IgO%AR>}pGf%`W$21cNK4St&O-Ht{7l?5BlLR>eao#AwyXxkWBMtWiwP zjy9QHeE4Jq7tu|&k;DljC`)Yq=|1>7xQCsSQrL!G7&37$i zOI6mDgVv)i5(G~T5zm|qVG!yMiZ@=G>JXB9j=}daRCd?~RzUV<4ZfS)z#&)e)N5 zo)bnjh&+}{iDfj_GGT8}3V!#b~zAhs!m9}{Bz(FA(sFyx#$Z@N- zAv-C@)A%^Os_>~}A>pN(nf?gA#%`ComG$*dL#dDF@gI0RmB&=#WV|Om<}*>~Zq;Yq zOqnxQ<8~2us+QCj23{G!qnk~`zaKXoNCqgXPL|5(fhL#*2PjoApR{Xu_#TC#MgIMA zuO&F#C?EIe)}ALszgDou&uHlk$*C}9+zL(n^UQjgt$U9pD@LN@KLa-G#F)Si6|g7d zx6vGRC|zvR2n$WbNC0KiBdHajiH0$WJmE7;r<$Y?6$Nc^RD_2<0w^n-|uN36}kB7@W#RJHsaa+kkK2b6nW zqFpV6eCQi?G%7E4rfGX<>tl7=MVc0L2go=iLo1_EYwdcSt8U7s7cZl;mIfQ;sEn(` zuOi1|MZpAc+MkdpYSjC*98)b-)z9Jt)+fzc;h82l1scfr3b7uh?Kq^VD+OzRl=82U zh^XBAcr5HxWNtXdnBgQLJ5}RGm| ztxdib8`9*T^=u%L)+CY$LJF0)d#`wZD6_LuHEdNl)u{IAfw6v!8B**v$#qm*b43T% zI=31ehwo#`z1>Q_ssnTGpnXUCKW)vTPcc1>xy(tO5(&N-p2MC()sgXIE zV)P`(+(E1m0o}wmyBi{V>ZmeNIa*uhFzKb9R#^GRnVY>|uCD2XuNPRpv&)zb+rxZw z(t?R1hH~t1ca=~w)Eh2}3Cx9h!EhNzq4Ivv1knVkq1VmG0$ZXvUGdyjn;OiRX?k;f zC{5sHzy?r6g)Vm9({to7OMF@0E*c|iz?P5-VbJj)c)muS;6DzL?pIpepcl)mjz}b%)REWnI8_!zGH&$`nes}y`X|Ra!6O57EMSqTHJ9X{ zY8KHIcF+5{xOY&PF?720x}48!T8uW#CrJxhcH@m_Z4-c27p*>bQPW}K0ZFQR76fDF z6O=&07ECSsdkQ5im`fxY*Rj^kQpEaoj;aEz9bW%t<5#@fUsANRyCt_4qJ&Xy6j2wi zo|u`a<@*i6c-Z=_bZ_14Alc)mN&b@qf?HpYwnrD+ewbrqAiT*DZl_3w^)OFG7KI<2 z637a5O(DwHX&SME7l`hdkkuHJG^e))rFkQ$3|JfqpOX3AU@Qi^ps(!DBqgpNWt0vT zqXfjzR!nT$QQ$VH?hGa*(i7P0jF$#*Lw)h(d8Cdf>HU@UJv0~n6%k=32dO=n( zYF4Grz(((%C>YssbKN_XoHp}C;U$?}+rJJp8IrEdEMtur9lcaPiQlVd+>MvNQaz8C}Yd%1Wn8 z#S%#Jp>(3%Vfu*vK?P1B$Z;{Kg5`uzyOt7$4#Gm>1|ktZQw#F;etW4y> z^efgMWGW2v0?L;uB!Zb3#U2P5$}C<}=;zNPWgZBSaS|BVpk+v!a)5p;6tFxbFzQbN zXL&s(j6Whoth)omPfpk&K+AbM->veTIQWGqQ2jvb5qDzA{YVJm;2T*f{b2K&zQXoT zl6oEb-KgM9K@fc#DuEc-W2A{ffH=JT;zBsnk%hpa{1N>G^i_ah5aj@%vQ8kT{ALgb zBdsN`+Y0c;KZf>zhrWcrTp4gn1(pBg!tdGxWd{u^0aG9%sB^XftGXfQL=X;O`VxhK zU+k2SU~jM^22i|WW5YrSXA2ZmCYFQ2(MXF9K?-1lK`Y=H3Q?W;QNr~Bu|VU)4A9<4 z)8q^wIr9tjD+LOyjjRFS0P>6^@XR6*0>~*6qq_zk9QZ($5b|Zl90XuyJF2qA*arc{ z0jNI!O2FhWQUD+lhYChSKhi|=8*COoTp+c*F?A0_{E@HZKn6fKgn%2_(C+Q%;EcZ* zp`nnzZW~%Kv`v&isXxhp+|x&01=tFn2#<9ky*OGM2K+&l;66~eFLXe@d>A2)`kmNf z?_a^N5ORq3{TKDQa1lsp#43`}NTAWEJ~?&}T<%ZM04R`fLVm$7mYZ#)Z=C;}nIsWE zR-sQM5Q`K5N~|LY;V%H9AK^fu?a(HNT=tKIo4O&Kbzn;hk~>?@WdTIbfT3BT;m9`+e73(N#V@OqM6T* zv2wzpnSkX;yqGyXetG$PK7S&3rl-fJ3#4KHgjAK6gY+@TbMg0P1px1XDEHb-SQm z;yEqmWU$nN^C%^}*WT5=0LrBASTQ=<5Z^t--D>I=BQ&^G?QZs52-^Des67gng#x$bi7F*@O0!Fr%1)@*w-MBzTeOH_D%-g`mt40~;q@!<6eDv&fXujiHlSll?*F3l8mvVFD+ zrHaO}EaSYzw~eohM;PI6hJ{)(9eI6rmNr-KJ>My_=fGc?h-996m(uG_PUYr;UGitA z+J$=pYka$AL$WjXt&!Ih8gj64gUDWYnynx zBgVFSW0GR#8xMaeOqau=w}#7UWrB9_`ntre>thBN<2B6Q)mqI4BSj+(TUCS8>fj#M zyCz{_==wMZcD0yVOd`-TD*XZ>ZY4;7eLg!fJEQ|I$MPRvy)aYxl%+%_>mzTOoDOU) zJB!>g&T4Do(^WP6o<|jQC6n-Xij+*bt_Sru?TtC&?n++1E4IFXAtfTI?#tijYlY9m z-XqjF5p(D~vRmwyr%Zi_VB{^RX((>|PdvXiiB2j}XC^EWThr^?SX(DK4u@ySM+V0G z?nTBItZL6BZ9^ygK2#%?+cc>iP!63l7R+2V$nsSfIvb^^(M2gJQyWKQ`G_mwte*ek zb#)>ttbFZsc(M;&Mh?%fgXPk#{kS31fn~Wm`@D6#+|n_6zJ?o?Lv5i+Kf_Hmm+wTm zXk{o)(ck+G7v>WbcE;$IeAr1*comyYZpTEIsIc5WQ;X)*(R2MkN~Xh4B=?X>XqA0i8jN*KR^M$l>=%2WE~khG#G*w7b) z2K|;md%g|`1?r-+ZJ#!j$a+90_f=PJ3Eu3zBsWc9vj;&rD!J!ut&W>nv8=q^^yay=ZU= zQlE634{6u-21GssHmE2*{GKBd=AkccsZS*)o^wIC`%EcdNCFZ2nUkx5DgCdg-kjh0 z1HPOqH(ix_@cL=|`1g77{@vaX+j5T%I?oej|NZC_mTtsGPQHF1qsN|uUYFWXEvL55Ox9(`hA$CuybBv8U?M_tg?KpoB>x)S8hL(Du7k(j zR_0!0djoX~29-CQ>Bu(jj*r{^$oQI5H?BP$ZK#e^qM1Q1KBT5xy|*^PJKM~y^C=9; z>1ci!|FCY-^g~9Jcx?VdylD8&CYb!J)`vQWf}qSMQ~U4J`&iFaYEljKaJx=8Rl43V z%LIXY*5MYC6(NI7$D`tL=CFSTXz8M=bUFNSODK2caIQ*OgH%qcG~dBvxo0kYr(D$T zS2Qv&vJG{NJ}Fn1^FgV%_dDlpinRF5l%U8pSwx4ofV5O0%gCWuDPg0A0~B1ehmG%T@*~st1GHv8LPmd`(#xlnF2bwG zt6roEETco5`#nyn&x~xv;Bl%g3~jnJ>;7y+6e)V4SBD}f!&f**;W)&(GthXcxMNAR zCExchZVHLCb05|9+$7`;9wwT_q2ZQ0{v)=!&HIM`lM5by&BLW+cQfZg?DBkwnXQ+O zHSod?&xH&d{he_Mx@f3y||NetRO89As@6hDOv|A$@>jar$ za}7ys9e?hax~+|fC@#UNm*P!`fS`=#W*W!&@z%Cm!uIt`;n>v&)i#vOfWtUgsrEIN zKY{f{=a#h?{Ln{oc=ZsQuZVr!Y(~LeRb4fg7(Yc_R6*|FXOLUr;)rtl4x80 z;RF9O5USQZQ~5hnqYM)oPyFY$-SAQ=TgQ8VrNRJ7C^^~IKIZE^TmOU0;e!Lc_wmj0 zK)Nuup|^41K)XYzHOA2L(0FK2w-t&wWSr($!W_l2WpH%}Wa^yIiE(%F9V4AwHFE<( z#9*k&enaOptR9=5&T}sxRrW{1Nc|tcud5yqLLa%8OmD`kp?rnHkeDeYf;y zDm0xl8!^2I-|=5;%>40=r%=y@D(~9jMz^NzbV3@?V(dSynI1=0!&i@!ob`*oD^-Qe zBKN(cU@<0L6pIziY(5`B2EI!#fBM~|_4m+TJQ#PSLZeZiaSRi2A)a&Mq21Z$Qw5-n_@F9(&w_VpJs4%;U;xx0CmP0 zPZTTHI{zN>;V{zqOwDTR542?qeybV3wyypw(`1pmuTCn&2FW>;_r>p;VlgaphyvbzX{1-6*GxLA8od4cr zvM?|*{;!+NM0cK~^S4pq7D;0n!q|hA(I_md1YTp3^Y}k$Sg@uc8VsvwCiGH@DaQ0t zPb6>X@Fvkj8WGAZ<2?`AT-lCX+1uyO9-r=4Z{9k&Z}(nT=kbWljd9CjC|L{;U_2W5 z&>>}@#(GioP=5Y|04OLanpMzyBN!5cXu^a#u&!5xGN~cJ_(L;(f^@AQ;tddpBzPVo z5VTV0;9~NQACU;ss~VxdL~Vb8c049R-vFBA!;l||6I6~e(b8~o75w|3PcmA_EJkKv zNgy)@H`w@^Xo#$`K6YOSumYH)ObRfNzyV-NHKtW19l`$en5m3Y4HEA5xA(1>@$vZTNo-rg``5HI<3Q(;f|&;66l#4~ zr-B3T?Y6<^-q8H0F9*jw18mB0p_)UJBTEz2l|Q+sQ3{26pVNP{)38& z6;0~y0Vvl$;YTfaXD2oC8s{gb*!)H;AN2l``>Itf&iyMl_4N~7lyJ^EzBiVH_o;Vj zTK#p?kf8??)!Zq~kIQ|r`r>RpT6|52@9x^efObOFN{_O@PCg29kR`2tRryc1z>l2v zij1TZv!%-ljbEcd`i4Jx35y5{_x5W_lpswHcj5L8SfV&yp|ooZDyHj)$Q})`<{TbO zoAez5h{OpBQwo8g`gF#htm)s-I-00xoTF-Emp|N54L5WgnIiXh3%gUCY zStxAUm$DZt%*>M}!@I~ir4k*O(4KrV5yJs2Rbkus0g=C=2t1QLMl2`Fk+^;23RVDQ z5MoMMe;^3>5BG^@Fd{sE&-a|bnEV*6>!*lmJ0`k#V*5?7lzYbY+qjc>75v3aWQ6uC z@J;7BNet8jYdKeyYWabyE*inop5L7z7jsiLDS=+Yx5ibw!Pj9B{LoLJTMCLM)l?Iz;Q6(>X~Z9@e5Azw6o}kk;vm04-?w@ zmZsCInA!FaGm+xcsqXmO;=9T^-McvzczpkX+X*VN6eRIlvGQ$w6EKe-n12dkc1hV zp6;IuSD_(n*lb@UVH>C(qA(_AdsJiO#56C~;M|x@xVtn-ICpz_J*;iz(Xj zhPXX0>3O?Qb*d=|u+nQhpQ@cGNiwbA8zqfa!z@uwM$5FSk8*qy%~{VEx*kC8 zs4?f?{`;;!D0G{WKd-iVoYdz2Y_R4Je{BAAAu5(nk|KpPso-u$mtr$rlx@_QXMD>h zc#LCj9m|I&4>bK$+oi*l)~sP^9)vYjuZ0j*G+Derzi@ow%N1xhL*SmWgltP!&P=p& zx}{TJQ_9tN>R(G2+~nDv>vE&W8Pd~G)X?}A^ElXZTHj1P&=zpHA(oD!nXmf-K}oZc zj2zgKkb36O7z3fW(M&e>mZ`vEgh)d>>p@kV&%oknQt_)_!$Yv-Ts9inXrRyTP0&DU z(|V^}d4UinvQ}D>z2;glxl;+uYMLN{eFTdZ_~-~7Ce4J;bM3MyL)LHHW-_T#K%>{W zcPDTaU{^Q*|6-%Obfsj8=K42~+ChUR)V|$xi{OR%!ssADJYsb)pqPqdM5E9+f36Z~ z_AOlsJiffRhlBVU;Yo11lLoCUx|EjSU0_Y6$GL8;ZCkL249|U zol7}ng1V8H)4rOzDUNc`?0RQ4lebF$?YG)O7$G+_D}TA)%kYYJ%a~XlBjNA-vAdXJ zyd!PzLXYZ(+UIF7T%KB2F7pL;c#~^-BK*qp#97=Cb6z5*h>HE|iYrk@k}VJ`J&Sha z#IYNkFxX{shU>M|P>#EF{PmP&knT8jI-;$#d#4-9E4yKo#fVLe+QgD3IPFe%J;2_R zGjyk&oCoT}&vu*a%%EdokcaC|M4QOlQFXo4A=%Ww?smh$?a6B@-0+wTMdKm9sO$08 zdT^D<>k3-yUrQ7OsqT`|Di}D?l@b;GQ6Lll zHa}E;y33f4I-&sMrJ}W~J#rZJcVKNMM|oZ=Hmb)_*RXoh) zY9-}4Rie(_n^@;~-!dn8R^7t9_!~E~7y?pK50h`9B5OT7b+l~s{3P0KrIVUMy|)Ow zDs4YPj}++<*Sd9WL$R~hm=X-i{CpH+Z;bRxx|jA>S29-&{vxsBu!6&Ssm{WF$SACi zXFy7uWlDpk>6#98w&stQ59Vn_d7J#pN!_(M%SFiO(P0x39QSkLkQSbC~W(Vo^|s7yAG@Itt^aRGnO<-;{GbeBICA}j2v>ZrbiO8p!P~{HT96A~Vvu)Ymr50yae78@#J@XS?EX6R31wOg2n-{M=(O zlmB``Lnf<1K97G8HDVa+n=~Wj@>V&hAJ2V+Z%ax4mSca4h;0luUS`r@b0vG6!*@JB z(qK7NuiIog`$Vj_aS@Q2-%FEQ7pM}#=MoyE;W)9`QG~oal-9Xx7U-7_GAOoHXq4_M zn|6l!Dq3a@Ph18Styw)gBDTy#y5yFQX7^m*{0+T>byz&rE(w)KtQJ}o4`2WxTWD3LMtq~J82EMXq1gMilv9Q}n8cFi3IG%j^&aAdrj3?5Z5{1Eie&lgjRt;`JcX-${95S$!`mWXy0UsOd| z2LU|ZZt;@M0m6)5@l&9uj>UIw>)3U#KM{5^WC0^WoMbDi{M0ISgz~8;I(61hX2j>W zKyRg9ND_UdBW)=h=9zeNmA~gltJ@vlJvRD;Gn3v}H3Jwp0Uti+5~EK}oXXRqpqa`i zXz?gvu9a$G$TicFd=n5HA$lDHRWiqOt9E7RcOO*Nz0#X z;nhes-KS$diFGktg^ROw>`F1Cn}QEDXr(0lSUqD^jcsq1_+UcF=u|7%$o)5ypd8*# z#7Rynes#Tw&$^{?kn}xjb5Jdor^7NkK2_v2^myIwO;$k?NsjT57Jns}phl$(SvT;U zD_SlWrY|u(-^pm2PSKAK_K=Uei;#Iio1|pZ0ePGJY*9qLg(WMsrS^U7EXNA zE*2ONPsAUDOHKGVe}IOw#57D`EpFIm_o}D$;?qb1WL384DKEGY7m%WX+}rY|n3~K} zo%tkVfpsRVxD~RbudVQ&T9Tt;(;kF#$;YG9f^ApLP3D9Q)WYRo*vHno{S3>LXEQ;F zn9l>#T>UWQK)J0dD0{3D=S4QNoUlu%!azz_B5cgrblVRLvyO2O0n;Pf)4WZ=`<@!X z@^Jc2=Yx`n@k=~aDiz|P3ubjRZ3Bf}T(`%!T-OMY`%E4VpKd>8*gPD<$VY6{e07|2 zKD^^1^KTydNnYL*q!#c$?dwreGYzORPBjHFf{~v(=xuKnf2=i-g&${`E1kzR8;o7> zQmkUG%oBr7)>4y~<4CgyRb|-WOU4HU{lsuCr#`w};al4)5UrsS|X(1=-%?$4YjM%UsmDjLTR0mY4AV%0&(Q)f2m2D$bs+0FLFuHcTx zkksTe#GTBv^=~@BeOSSW5?|-`C($vX!1A2sUsUx>j68uu$26|&-odKtaUO|qWP#?7 zHL?6FJ|4Qwo{N;-5{&O?lGm0k>RfFenTVUp7k@)v^Vc%Q@?vM6ZsMISJ52T8;%{*2 zlo=#D<@26g?ehv+MEqIiP!AB8Il+`Pc)j0^PI~Y-!DRFdI!gJNA57u5Zgx$26+fw0 zI!-TwGY|PjVugR$xPgj#9V$1_ZPeN=Ea@8mHTkLS@mFfm2%XyCl~7h}l>I0=+?gvX z|Hb%NGjh&-#FU?dla zopLPAaN{0zjz$>hkjOH8y#T!S9Czgz|lm^*2W1yFZe$`)Y2A)CJqKp7PdA@ z1~!fWdSOWsC3iSqbaHS9 zPzV~^8k&5UC^_5N{W7usmiRWu$M-)pDcg!mipUt)0qFmqrT}^oH2^EaH(JR*y$~^R zG;*-8bFy^+Ffy|JgMFg-k0L`ScRQ2+q2M1O>6FWN{2Qw1`3qS|J$ox+=2PYjP8^bsG;Xl>P%*=EQ4D1}=wM_riva+(# zF|sl-vws&bvjM)ZY;0_FoSaNd-wgrZI@y_+zMHdiass~5RkZ$7%f`s?ZHJSC9l-uS zEwcSf_8(LKrRYC4IR0aul@Y+o@^6`cl>e7b9RM50|EqWGjNkhI)nE31y8oZi{>R9F z?wI~(-~SmLJLf;*|0>{M1aPqYOCJa8x4^%q=X+%Ty?%y&jZ^1aGauhS3(nTi68f7- z@}CIOKezv^ng0!R4S@N7<($mYNtm<=WIz;s{*2IIiyS+O{{t@c6CgqJDsRE(w*uO% z9A%OP?yI+(U%mh?cFg2@H3mwbS;4pl>g$^LYgV{6+JQsPl^n^2!+fv+GbzQV&~Bo> zF)CGNk*;yHp7CSo4%g3jC(rB}-HogKG=zI(J4>i^3P^6bVUuxdtNLREDH^gr0tEFb zDwRIl`%(-H|JR#X{_Pt5o7L*4iWeO^Aw2FZi-RJ^x@^JuZ$BCgp)ioUj+*m;e1)$b zf^p~bL$zZeXGH-z5t;}Av{_1dk}#?Q8upJT27y$lF0F+#`loDWl8vB+{7zECx6 za9gkZ9@Lba@{f3t2_BLpAwx~Y6|w~0YDtneLZWTva13NQ5=D;`>&7{!bCqgp?+d}L zY~lUEt-;OO)*3iIA2{yS#b!SK3&uj+|JL7G{`Eur$KQqiJAbDYvi)WJ|BuuEcP{^( z%kux-<-gtd+vor77vFBH^}k!8tpA#r|J@2@VqyJv_%llhV@h)oeX{*|CsJytpz5--Nk7JE91WmSfj*wwU4 ztLv*DiZVdo84dz+@<^V!h$)+jU_Z(0|R%u-uL+);QBC}7H? z-woFXE6Diu9>HEx-L{JWey3cvE&(nU^oKXb98d$N)CM#oJKlP+tWHUc>INEOBuuK?jJFqC<&u z!??36arfzkL57Ej&4HVrK{z!Pr!N7}Q4BnKp6^db zlaEQ#D`*DrHZI^PLEk|?-6MfgUx29}%AaBUgR|FXEFcGP_4YKbwvS8yOk$t|eVb#O z+gQI|7pYNSCDEDeP9RY2Ij}l%s2Y)CMM_lI?#@e&*GQK4>{M*PT9UM)$a1d#O&wZ0=%!o$6gk{4Q+pylqw0_EMLcr z&W|Z;a0dayH#40BJQ_S;+>5Uktv2otj3Z#D7q!5XuT`v&ULt8)T)c}eU~~Z~5;ipP z)?P~Bhc8zp(~n%7yRW{mFL8)>ex|K2-@%U=tPflI+pq2C7yfdSjmryV=}Z4FXpr}1 zUQl_EcgIBu;w`2wSzA`1$R%iEo@jeX_;_Mzk{?^&u z)Pt(GeNOD^AMbHLD29S8Mfa$Gf`qg2cY)@=BDQ6hTOS+_VxaJGyY4!G6#PY zgZ_09U6NTIIJ~X$rWfL?@92t=+KckJy#CdC^TqXLb?PrbpT{}3)==0kUnb+B8luJP zvo)5K)Gxpf`5=*|q>-pijTf?diN@+BShhp8uwHhBNb-g zkysdzEN%dRf3ziM@x5;v;n=W=GW}8YTF~gYR+Kop?IgTkNIdA>TDJpgShB$(f=#I8 z^-Yu#Va#rek{U{O@4AA4z_+|M;`X;+j|8Koh9OW-JMK`2ReI0x&s0aSkFHk$w46@7 z@lvCFdvKl*gNuD%kILa#r%~mqQ`bZLkP(M79<3ouK7GUC6k%ejNeSZ3V6Se&W)PHB z9{6%*Cpwf`N!~L~EwmTbj{lnBjJAS2BWbXoDsyJ9u=TrF`q7*#6w~V;9#)}<0!`vN zj_}!Tj|75S1^!rT76q0QL!A*%51?}_%hnR+9VZ=UgZpahFi2+IcZmcZ<25apk(5E^PHem|Gmf8rAA$xEgew@V zET{GT{W*i1-rq#a*sjU0C|!L~?~splie&Fl+#J8=vTxS%wc=hwtP@S$t|sN$MwG~2 zO%aH5U!$kzwAewL*5r&_p*Jl-=z?#z$NR$@Oq{ZSW3Yxdq0#x{QxorBM5%j5D(_e! zd@Ecuo+yj^72C1_!4ZoAgb<3M0!pS2<0+jl-to_DciUYP`3-`pxv`KEV%FYKY->mL5GD9# z#9VD(C)>ekCAz|(Rn5WB3M^R$>VDs_e|hnZ?TISlKS9_@_nA8vET(psk<@Otv>f&} zKbv3@VuY#p8hUcdUgpKJg_fx;*}Pe`UdYZS4m`Sshs&mfp3DgK=JN%1>|@Q|H3FGn ztX(KY6@AqYS{OC?ho*&kj z{a8;DBGS|ATWDJ^Fy&9=&(m9Jmi>i_q6Yj^CSHmQv^DJ7Cuxv;9d;x0>CfjMVnMw? z)h{SB#B=l@k$||{XPBZ-bTD>^6pON^8ditv;LjCZ8OWyuooCDt-CeHndG$EhU7_~hoB`u3P`;Ac*W{DMiEz_&__uy)T1+;+<`;2HS!DNNaF9V1 zII6R$Z9~3&E8{X1-36i{!3bEzY_H*FWQ{Gd8|Q{;hY5EEe|5I5LGn(JM+edRvX)hQ z_elbrc=y=tpDep0M~3pRxg_yJSG}l(HJF zzcS8GykpHf?)%=|NwitGhE$)Z<}m1SJl92XO%~NTo0nAsD)o$*#2Dkf(}bVjyOQPO z1T-C!Ql9r=vGt?xCd40^mHUMRW`o-&;LkwGy93I5wu?G2u6aX^-{4m;Qn(ZTNELoF z3-1ZS?^MO={0jD_xehJ*hSAK(KvM)9K~$5)j$a>KQ`B7A-Lo`AVbz5GB>FPRdEFPY zZiz2nH@9y3h!SdKABAWTfPKPnu9wNU${?9>G|2%=gL#2uQc?87E5X8PEjrT7ihu5M z0DYp&yqE9>^&xelTO7RLU`)tr*-?8PE<)g1-uN4}my(8#p!y6XdaEc4cm*lVJ73*bEy`s6!+=1_~d<_nxZkO zj%IxgzYivQoT*GNS2HT~yK(3idD3?g5tF@KC2HNUwfSO)agJI&lGt?)>}A}iWdb`7 zkWge*d=P5oE3HdI&gxSLD~3_a&X6l8cU;xnX#|`5cb~mAHY6N~Ks->U`l%}Tl>`gU zc2|ttD(vN066FIS_u9oZ&e|p`G;$@DVae8N@lJGoovhBC3X0R8h8W@Z&%g|BCYr3l z0zJ4tZiMdK$qg7F*wq58!7Efu+49StE#*CKN?n=pFa=z+ab)}9C}ZOU&#EQ*rV_y) zDqn$>V)kb?SJN_j2=s4P@yMwHn7MX%@85*-7_LNbJ%n{x7=@boYq<}CbnUlIj}G1? zBT`-r3t-JO6HYP}-fs8yNo3s5uy+u-BFE+I$V;X_4hY*5_$G1u(nq!mLUe31NDZgc zsD9(cF4YfFb=oWV4!A)Li(K6T2i3&c-b{r{{6|HAt_$`Ml!`{Tk)U^Y zY3?L=z4^&)ho=x${Av8xFQ(XM-qQkiKk363o88wr7$d(KN?k%)>Ieel)KR5Qnp!X& zoEWR84Cc$Q&0{d=UPnS%ZGO|*r(8~9{=_Ok64^BGU2Y&NIXsP_$JnczBd_AvCb%}Np_2%i$7i68l5pmDXq7;JMsgepY{FMvWcSwAJLA$>wsMjDg z=yf2rGo$8GcT1i~2KeDqnHN((l->BCjxWqUdzH9k`tq*HUC02 zkg-}Ht}T2RFQ6g_9tinvgjTo!s@PjgIjdyMgj42!q_EB(R&3PqbvCyR!nB36419z2 zZ{18LM~KIoeDA_N2Jb~dlt9(HyVv0q@}9$srlSkoLVwM%xpv;MtkJMxJL+;*)EyPl z5zvd~PgbZiE^WtWk-Q1sZ#{MlCCACmd9}WlplY@Ycuc|Iw|?KTfn&nBy?AeG2LGpQ_{wf-EoKWPF1ao%H>m~MyoI5 zlKNWVJk;I`JZkkudcy9ex^FP@D_ia=O&I$;|9I$=4v-%z8m`!% zLfvAhobUaD#VmeB1SCJUrg1v*&afJd9BPeY9lRWV3e($wf8NufbQ7PY7Q$i{d5%_kliU2P zqY!juZ41VwRBv$l_2m?^0d~PROys(|Sng?}RxU~Hj1pP7pKzgM73?ATq3NMqMzFyt ziY-1Jzyy3ebZK8YF>RIIt7jcV0dZ`C2QbG_+&-DNT)uYP4wMkD>{Ac1P~sZLI)GBV z7?BsCEFvYd{5_%Q^U7fB!Ys%P7FUTS@9ccno=ij@c3odG1ok=M$*wd7@Opn5KTFPd zr*{Py*Hs!W1;Za91d2eUhZ}#dmyCygRKj(D*p$*^X=Wg#cqPwURVjv^TtCZYTja5t z-+0?oQv=5a-a`;@6L04YjRg6gBBZ+XEt$R&IOW*G+gR~){{n)v<_u4S=mIPK_j#qY zB|CQ;x{kJ0)=eQQ^9Sv7cy-dU27VQCNjakZ!^Snwm<40b=#`QqTbaiiXjn?*yv(1( zb=Df$hDcvcO-z%Ym%`X!5e$>3U!MvW4`mfVi8FRsE`lRN8Xed#MoQ+_;~^C8LI=ky zQ;+8M3ug^%-%^>916>Vz@o#_b5-vKH%_Xv&_?;^B^2VEv>>81$vOIN^AqSJX8zxa? zeqmraPS%l1xaW}A$H@wP`dMvC4G$;v2wx07^tGp8=J)14O@t+Wd@1rB^x2fRU|q7!y$-yECf!wQ)8*hFag1mn6gy zO%0=!jLG@xjqxm!Gw-BV8H>dFnvEpq)L@Hw2`;Rt=4WlDt~}dg;ZdL5HCVVT7rToJksNr>7)_bIS_m*u zR6}q(+9RS^oF@#s14__Brl#Krbv3t65irp#y%#tv1l+AxEc0>>^8 zfno!->s5p%AGKt(ely+O^UajomDg&LNukFWI#1zu*=hT$4jLvTNpkfj09kZksQzfQybl+V0BTwcEBM>Hi2pVQu+X3 zeH_Jy`CvJQ$Vy#@Tmzydvnm_GE=*V6R3xipDRJ8>k@5pZbItJ5A_4zHu*G(sr1g{M z?3@a~DE6?lj%>fxwoFm$#<`1^dbk+E3aFNtT=Ez-qPiP+rmWo!b3ye4!QGNatSBhC zP1Q1l`!W!UD*RX7F<*=`wHGfub9`OZ-fKh?rMOOm-Ue-a3m$F=AX3H7H*zt-^|)Av zaXeVq9Eo>~{a|(V!t?lPl~r?|s1a!w4G|v~FF3gVf$@ex+6WWlEaG&y!~+xI)k;!2 zY&1fK_%`^j`TXhnK0fOoNw;)wNsGZ85=;Lz>rmw|yLGkssOGu8VZ{k@b7*gHYCv0l zMV*Yfj@+Sl9S&>tLhtaYW`Ign{}PM}`=OMLvWz9!0?od&B49fC&>cEb1=3OMYo2qT zxF`FSzb3h6gM%{*1@!U8Mw~=|o|`sKNgMIsHmzH^A#OILsm(mLb|}NADy8k5k#s$9 zg730axmTIbJ3aZHLQlr-)IS!@`f!;O@< z=Xa*xM602qg8-rQ-I5pCPOpW)q(va-l?0(Ry{?=4eu4OjoDt(UH-ie|&A{F8x)ga!DX1B{H*Wpr2#1 z>yTX%hm#unj6`qKJJIx)io}-2mCSh|x~?lgy3%dp6E04Q4kiKUDmgOfE)vF0G-Z#3kw*9pMc2vfvj266 z5I#PkF~7fiP1LpXfIep&ff2!}Ef$%EEqx4_EQWF;@9wkyQk98zTLd8EwvuLS2_Lgp zfUzN)3&4402C9Ql_X_jVpyK1mCcA}5qW^3!wLV;~du_O7wDD9KW5yLUA>K zh^diwPZXgM$mGhK9m0H?!&1|pEi?EQKn&~)m;j-&7*#N)ZutLYn>U+;7xy_?lWmL; zZzP)~7sQ5uS!7qf!T;7iEZjhBC-D+bK^+wgcVArwu@%WCOVY(K8`TMAWbKH@jMQte579~a5`i1V4Ycg7ky^$l*a$EPLnCA5Owf*`2?HNkZaNpQR~DkasqrF4r{UJW!|1;rMntbjoA0S zm<>~AsetDlE={lvUo;Q|VrYF|=79=c3ucd*T-*-_^m8b$ulIBcs~Gn$#8-`~qUVuY zYLYAY!lDFE%lk>eH|*kbq}WsN4A>+vMVUwxAm+<^VWSNAbbk$Y`{qt}TW>D(TI6zoG3cON&7HNWzfm;PT8t^+k;7bo zu%M9Reqt60$Gu4jzT~9L1zIJxgJn9WsvIRU+Jb`EE0W2TB-N^`a2DNQs-m|>Jl+9A zm4=PPDU~!7vx|1{s*t|fg$hHsPG)g6%Z109J4vwCsUaxBe(JoiwtdFqLfQ=kYL$6w zUQ5YEI4#>`gGzrfQP8kfdI7DooX9yp{3Kdsf5p)N6tvx& zWnzLX+^r}!b}CLf=TPAfWT&9+W|A2VX8OhkDJy(Hh^#V+Pildd)Y5rG2>ZvR8zU7@F$jrpgv{# z0Ju0(p1>pS7^UTyUQM!HNY`b6MvbE$ene<=@0+PYJcPt#Jg}S=x$_YzVIN-I#D&S= z*7GodfLpnjBbv~Zs$E1`OOCp5H!}5JN)p(p(VMh;MUZj%Ccyu6tJ}!!G$M*)@c)GG53t^yB^=UkRuaDYL=U9A z#7aqjaPWiDmjVxK57{2Fc`G|C9w_CQ&Y4SA|3%xketP^21F`Ce`PI<|5_`T~L@M;o z#omQ6*A=n$AYnoN9;%RkwGR54SN(9~e&|$n84A)i#z11Bk&z;!al<(R6QjkIk@3rg z&vI@>*Of0kGJIR7bpDH9c>%p6FI@j%n8QA5ZjM920p1pZzbSxLCu>mzA!gIc&Nuh8 z4G|^g9CqPX_rW+2n4wU1O4*uH`JOX3n8V3to#*v26q=$Aie~3q(qPL*Yz+kEX1+H( zIg!JAs*YLBuXAuo%=}9O+AERa{*5`R^3P=<8%&^rqE6W_I{l!+L|8@qyE_Vo;GriV z43qR-<3obqG|LezDvUhjx1x#P{`Us2fZ1RkHN+q#85Po&V{HtIer3^heXY1&q^IX&&5v2U{er3~*{Py-)5tVr zETz1Shg!i*Y@Wj^QY1qZc0U3YZ@E0{{(Wf)4PGWIvkgU``FUzB@%Mnh8ty^bPxY>` zkRg}0D)KF81;{!D%#Pfwa9zN1`+BZROo^I{Q&0LHh6L9C4~{T33{#UE2NPsUwFv@U zg~cKdpb?U8pVTkAxcS-5DSQ5@#*6q!P>Gg%aV2T$c?ACaol$Rs14FXp0YEkDVDw)poE8p`MtRV`S(FMokAgjCQ>sW} zB^nO(?cf_xwTb${JM5wnnO{rJ9~o^!xHEE@v^)Tys`HwKR9fM;&CB4_N4 zeKu4#Pbn=&ggyOFp%8N8%hoa1Gt}8P?d!k609HDct0YgMs!)RTNYC{n6Z%T{c$!w5 zS7fm#V$|2on*T!b;QpD=)TLL$GT#Ryu`?`ZXz7nOZ|YF5}DSNd-h zz^n93wd~}G<21vZ=10F4)Y$HL4dLa?Rew>pAARx}nq#QcQqA&JV-=T2W9(aqVuV0; z=?tl$SMSo(s)LvcMlQFD!F8*We!ZrL-`QBJ-d@lb+Uv@1ov%bykp1OdUvvOkK&yS; zL{K;UCPcc8#Ho4$l^<%MQ5mA3#6CC?2-nK$=>pAr~ zYawiR_{i$6@}@>GwH6MHk(h_LgB8iQjMcRaHd*$xYuhMm$D*{UCho3|K?s>t)ShEJ z)Rq~+<*)!z8Z6p-WM{`y@&2Taq}t|Sg4ostjY4~~og*%A49Xk+ae}l%HzB!ks4RtL z0$J-6jJq~y@1Ya9+EIPV?>oH~b=+q&u_~}nsJ8nDuAX>8R`B6K$S^QJ)3{pM=SvK$ zQ(C&C73|#k@K(~soOmE@RYCC|#pEE%aD)-jce|}jT;vCt%~e5%hCam?6|6KoZdIV3 z8l3Q5+j%`1mz}m;$oj~FAo1_al&d67vcBS@mfSCwZ#MZZ*d&WQ$Qku^wp33qw0NdC12c?jmgD8%narPxM z%JR&nNqmW)L5agHJIi?BYgP5=g`ti0!X2P}6r6kR!ZJ&@UPheega+$+X)qPyA{60! zbjhT1G6Y%*joMqyeBw&vJ6tpS*JU(yHfUN>dIQZKbN8(Mya+eyNhJ(C$X$0KVxfnW z314*%SIB=yck<1G(GOta8gf zQR%D&OmcCwyLi~FsigkTsI%3;H@e#yzvi&4(z6jU)C3-l35CkK&|#2mRob8*5GL+c zsT-Z^p;Gk|DGn3)vS=TdXLgVhZ5yBUdVWNVe|;-MPV1t+9PIt}R-TQ_hC$_l<*?vC zy=C*EfBH1cSA?4LFDB1EA*^?sq81*4$jfJGzH}5F4KAt9$!6Q}m^Kp`Pc(f0k{egI zY1tQ*!bGcn4t}P$@?B#;>-pPAlg>IqnnIa(03~@{g40{lJq+gl-%60PTX7c3ep5@-e^Y6UG-iWPy z7>i0h=q!`1a~O(ZHVw&TAN4AxznvyvDu~al50xD+Qgdf&SX9YLA(w>MnL}8;}zLC=puc^O+nfcj%EO2Hyyizi05eyzen?_UNAkNN`s!X&)onBHd; zd8PWvKy$A9$DgNQw*4|qUS5bH8F1O*zk1jmctT2@a?L3)*F}?OS{N}5(S0nliWU}; zaIDiImVgM)L-faeI~{b&@Kcifn|EdEZ_k?qR3wjI0W49$z`lE4kc;kSWD*u?O&-Lj z-<-ZiO9k{epycXg?i)@nl9f7s020itTMFC1?FMhI6Z)5Ah=DkrISLUg>_Ku=j0Wu- zDaygr)geU-1J%1kp~f*#Z|b4JlWKuKV$=1Y7ph4lP}X!~#rn7(6d2<8h^!R!ow!^Z zrF4t7w_k3iY&d8>)ro(AGnh!^-d#L+h&`E(#F=X*@P~NN*tk3Zx+{qzVIMQ(&z*>W zT!LG%a0ZsZ@;HdG(}+&s;y^&Hbps+!MxlkgMR|A@B7|#RQX5TP^rjr5;T!5(jhRtF zFj$OQ;bZ+2egi8mMp2)t6Fb&~_~p>ke;)AilJtHnuxl8@C>Ol|ZDIK0jTMqs%gA>` za=^50@rtCECYdx+r}1KvFAUK3EyUY&8=4+N+rmie4gb%Nv4%=EZg^X}Z!^68W|XB; zaf@W?4Zz|MN*?3(=?c9Gch1hXf!)wFkb48tp4vXoK?F7A3Y2)%9q5qV?~Xkz3lF^jVJ0;lqF?A z`*&G^4?nEa!SBmSt!7}!#n%#gwr5KeF;D`ebEmwLVfA_*^o{+onGKskVqjsVq}t`Q z^IdFd)e3#=;Nu|k=zs{4aFy7;ZkgfTS_ZwLBz#JOrgJ37S!z==S;cB}Dt9FBO7S5v zkc)i2T;mA?XUQFZNV+RL>Q>#>wq2K;CikVF>c4*KVA3p6_Qz=Hqd@0KWk$`x5rJrI z87u>gg@4eS;~wvrr6+_yn2tV|9<1~2vw&-`8*NMJ480Vu8sV85h#nSZp=_Su>+OBee5ap$~Wg*Ff z3QlDa)sIcSx5L+}Ar_|F?LQogrd=%m<=F6rE*ISFrN{Hxl8becLHCoyw>>4QriJ_s z2CA_A%~(Au+9HT*Ip7>{oP2@GZ&e(t9Awx%|7W4Pu9vc?zOWI&Tp|Ue=h+ zr1ZSn)jdLT$o2a)yuN#LuXIfthA3=vOG<=;7;b))ekpozgY6jqUL{)iT2OYzV_d9j zvs0u>t%n8vxX@!)d}H-OP51|XA`Itfb8O7%z}Tx2L66axwc26KiKsZFHdnRWg@t+= zf>k4am4H~ZOM=_qP>EJDk^TMS)-i>kp}TL|X-o z6Ua!Q3P zCWYh|tZ#5#%;h!iH>i;QD&4Q#jN;2B5Waf-FS>NY8Ar%e$wfufJMM!B7r(V|3o7@T zMXhmUA!`aV%%qKtlhLEhlq@8g1OQO@sOYXI&Q09z%BmPAWw@{Q({Rs|Lg- zQ~artZw!#8BNym&?xaC#nWw5ThCe;$)dJ{)tlU!hY%7w;Q`_ zf6agOG2XFX{e4n}6WFZt$h@Re->Sf^DIy}_qR{2JCVVSh?zRUBf>x!4b* z7a04|x@$JM-1EsNUV{6`7G_#)@a|J)4nJmm-}-}JxYXxr>8G#C^9 z#JyH4@vo@Tkj2xbgdt?$o^<*hI&eTgF~e9GRC-YHORyAzq3NK|gErY%<@={z_NkZi z0j=bi8|L)#9S4cRn3OXsU0kB9x_Tmdm@OB~LQ|FHpVi@1@PFEBs{h*OSP6445(?Z3 zyi3MKR*=oO-S#cpqVULhlT)(chpD+mA)A8!{t2~(;1V6YgNxwR_Apq7ZYx~#jHNEycn)2ZgRS7g;MKI|lsk9!X8CX?7~yu-sSrG1)Vf; zUXZD88@e=3G-EGv5FXJ9|A?bU17y~8H}V$wI!O2)Jrzam;+gW)r>X1BdLcIz4cP@& zubM<)D%S||9N}$G-^`0xq3B;o89wa>1)_9B-DF=0Yl36KB=bq>1&_6cSru5<>XI>( zSL^sYl>>qS1nNuzeEFKT`4TUC{MVjb|MGe#*y54J;YhaM+cec)sy*k{+CY4_JE&p) zM^xgrJmBPv5E)IGj|j}IZg zCW)jh8M7emplfqWzf!CR9wXzsR^?h=+7qG`R%R^yxE;voL8QRkx{;zQXo_yj-%uBb zGI{D8wI>)Owyn+%Xrow7aPvb}J__r8fU{5)sW@r}+*UYei&MZ){PLK@$E>+{j84tQ zqT4cqEq*UjomWjZ?+9-M{3O6`#nZG-gvbn@EH4o8lx8*5yGbU+poJtEP z&&G#Q#KYJQ$*y0<#2 z+2uZ%ZT?In(>lcW#8Bk0^GF4a^*gxg6FpNDB+-glsvC}7HEJMurBu945PSoVe4uDj zs8AW;QK)SzmcB_TrmGdA5mZGM6fZEoiO*=dnrhm?+Hsul5YH0a|3}FNt$I%6~`ZxXuvW%yGFC25Z`I+u4T(=XC|z|)m(re zX^&wR2hwwp=xM#T;+8AsQE-5TVTkWva#~XkWNIKwLCLa`xls9(o!sF_*tT39Qi-bc zs*}h(1R+odgLec)0dtXv!W`<0xNA-0X9C zRcW<~cDyoCa$4-d;+JhVlq*ou^m%@-)_F6tq5+pVwYpxboy_;n{O@6)GTpiaqX$W7 ztPlLsM5LA(law(Lp!-~7Y%C~IKx8Yw;I;9)x z@1mS6$dnXldC9RtL@g}tC0yjv=ln^+r9?zgU zDp*WT$8edreV?d%ys`bXx0LKAk?0pxD{2ue+eT00QBj|6F-#8xW_vzK#KQqPL z_+ew*>Q+=O>~8T=8Q$E>ggsnAXwh@XJxBgt(bQ7Q_zmtd!2GjZxpUcegJ==$Y2W>` z=CKGiHZ&WAr(TE{<&E)KpD;*f{50aGm zTvyX8Kg%*)!7F0?`+;ss-D3Nj2`uJ&ZV#Vsh_#5bt2Df7kQ9}K*(|gB8EiRu>N`W3 z!JhMR9^RL`)>{Sg9jC(F;~ZJEq86!->EKS!wu|ypCeEXjGw!%aml1^RI{JLHsAN!r z z3YR>Nx3I7 zEwpcRDL#0J>^PDm1Z@BTMBJMYmiI)*>XJfO8h{CZlm8xzzi?8R({2FrhJ965;H-R0%x_q8;KJ5&vPiixv6aU9@tSI2 z=5we&iE@-N=yhHpD^Nvr2Xnhl32Z7lR6ROH7)q<26%<-j<^T0?oe`<=9BfF$u2hm30C1u)^4Eii9mb_=yJ0*13G8lz@jV}*msyhF- zLCJzG8TU1+XJ2~O;Av#JW3)^k*0o_(9~`L0D^l-=MkLSnP7YVd?c~w zjWd&Lqc{(4xtHy+%Hbi=5rONxN*NBeBlG;WES*{RA6|%%BVgxEmvyMHfm6e*b-sI03szfn7bnAs>E>SMhKxdSG*c;o%ns~!C``fUX9=Ilx zv92lC^*4A@u7k7ni>|&Pqh?0rcY>dWf%fjc1u^sk8BwuWZ4Rt(A+o<%UjCy7^!l!) z5u5Xou_xl+ww3$)nd7^-B=e6M5?ugz{n~Z_{Qb7$*Rw~$#;GNW@rzBjlWISY(gzK= z{=ZG~s6XKKS{W`1BK;1VN2Kn_JE8KFvY3@S_ZLE$IBu4UOUmv)W1r z6`xBo!ofT0fa>H_m|bXrgl04GT9c<%=qT`uzy1S_Srlq8vW>COvYN#nQh+@nmBJRM1>P7#SY-|V$pt|Pt{<9G))3{sCC#3BM_f~%3 z-Vj7X2Bh8UD}7qL_Lv1uIMWuo6CdcvmY14HrwpF^~O~<|6)g`};YBF_KaSLs`8fU{FAf%wz4<>~k zE_p=D4IkDng;C~tnNLj6aw#N#OfBr0*zqlOYE~Dv7(@yzGZSdyk zv1;1i7y#VSDMhI&t#hmF8sbX^3q!lAiyWEUDh|_z9B)#73oumk`Fl-MuhU9)NAL6b z9BTLM6-Ix&ej2a9TRD=p=!2*SwC?jip14)_Ag%J#WyK)3A6oWPi_i3k5yad>_tla%XAnlT;s`m-`CfCcUN?`+{GO364hj zY?qE^L{EsM*d7oaB`N!B3)pI~{9$>yd6tYW{lss?j&!w$q&2mWVjNIck_B$BOUWPL0yn~OXz6UqVPx95Katk6>;*IqSWin*oyuqdL?i~nlB{8|d8T{hmhG}& zv%TWnei3|%?+9zPGzzdjaO6uQw(_SQBq1qmQbNR=s7W~w8+w819G!7vM6dWfh*8YX z)wnIL_K4-MdgHBe@H3bA8KJf~T@zz69@62@H8cur{UrH@p}`@@^H~Ej|8!<)y62El z(Wal5V~PpZ;X`1V<}lb7BRQq=#zI4Re-vk^rX!VLB0 z?YUEJ+^5yjrA%fO`#=Koqt~WH{U{it)4SHyJ$M=tb%p#>EcnN22wf^BF2*RcBkM?d}#9L0ZObynv8VWP5fvas!|E$#iyo>+P z6B7d)DEG}3HxuP#n>98m2?;lNMrW9vMaE#~Ft{NAVOSX9=-}jJ=Qfsdf_xT<2m&A? z+C|!uW`dWPHy{4Fr=7RA+Pkr@U(dCc)t1?(Z7=+@-7&sbRx5vNu+hLtI0&zFut8u_ z3L=n1(ERV7glFfu=(vVBaDk06 zNCF}-1k@x{z=_BRAQ8br!#_t60uw+_`*LQGQ!obCPy$|qU;&k2ax6$UN3jW@-@#v4 zT7Y?$`Jb$XVa!`t`(a}Uf1>T113ZR1e#Jll0z5ds>TAGQiC}KikF5zUfc;o|f9}Eq zJXyaWLCzrnSzB{CGdqCoA@qaOC_@E zb{Oz>b|~ai;5XObm@~NgumX9EHvU^Pw;&I%KhWdnE^~}T9f{ws z1_^?E`Tuk`3@z;i+}jKOj(qVC;ea8u0$H1D{(bx>2*@i-0TED9 zk-^5Nq5y*Y;En?xjupsgeBwTYRHC)&olV=clKQ=M}z-U-bVO>H_?2N&n7)Kv=!Ecl;EvSk;$wh11PQ0`W^e!cfV{2xe+}Wl>{FC8 z_ah#j!hSt%F9R#JVTWn}2eW^_mBABG00I3Bg<1eFqYC%33mnQJ!wC29A2of>s1@MzWB@Okx#@hcYU`XeOk>X$u@t+iJ?~XO+ zuTn96M%Ml-W`NUXpF2c_Nf+#W`4Dv^&uJs0w?_1+%;uXHW^1dQbt-&Ld7;uTEBTdG zwyHGP65T%uGA>$pVH>2+v-lK~a!cOpHY3wVVo}(H=zfG^fp!_O?O3URrFni#AY-H` z<$1bVdG>qy9l4f2UhX^)iz$X%IEi8$a9`rcnvaCkPw@FA!R5x2ipR{0Gu+E_K0J2L zKHM0Gt#W1JH<-8WR@d-fNi-k)I4=62LRPq=^OW_Z<}$rTjx1291xT^O+-zO~OgWL8 z$P57^1w*-V6(7og!@Lf;Y&z}8>c}S&mnL!{Ta-0s)GZ*t^P&48Me84G!46v6*of}2 z^8(j(y6;ZE^HfM9CNsGfE=3bt6gDy=xpe>~#4sbKQGl_j!Y(3Mi_TnL0|sV)kUtez z)uxSELJcU0)O#uD)3horboJLY;hYNuS?x3LS!J41biUD6as)B?>!up0{qLDdImyE> zNjs=?{Jx7(g|Pi+$~w?*DId@e|L+P#CL2ZD3pGnI8tf{-F?Gx8TL(4C3EU}3h5aI1 zd!QVQx@B?4;`(NZr2Bj~xWmi_k(7`}%&5mLX*qb&2oAd0l1hj&9J%GwF&G3qZEo~Q zr*umvwWdP4ZlnrIYL{mIgw4;o8Wy6i3_-wL7P6+Zb+Z z7IUUczX$h^cE8~*oKm{zEVdmUcC71({rX4QZLywxW6K7GJ{0sEkMrkhoXb!PsZ)u<_;5Ee<#@lmx0$;lPAo82IFqWEC?m+qO=OeW&HyUgH}!vCy`Ao}4&HJTPhA?)_k+(9E?9tXE6C)1Z6H5FFp70uJ<+;T>ut=Cr~mJTO7Bn$Le%n|WRBSw2j3*Wq3`n(U> z{k&^vdOK<1voQgy16@TzQ+Te7>~%;dwSgc#;JRC#xFN_7dWGk%Xw;RV;pxYeBKY8w z2%x=Seo4wJwK(IAttLc@AEOwOAP?xP_n(|IC;FjO?=ZS{DCPz^Ky#5_H4HvAj7f{a zMxU>OC!v1CSgNOSMRL|$z&=|$WbKTI5-7dfz*_}rYX!JABfk$(^?`7OCJ4{FI;M_L zys)!_67X0qYu9_fhG8)PihtF#b~qL@!C(@-%bMe2~>*tL1+O z(;Oo3ND{Wg12gg~VLhS=7AAnAusG(3uyh+%f(bBOEi1XxBbCX3p%OmsjOd8fRb#Z- zE9ey64lj!a+-AJJ661(hrpQ_^__xlVnK0*Zd!yxCWGgG{h@72A?bz!{bW1>Hkdh>@ zsRY5c50LpbUDfc95Pao=6H~ZK9$u)ZQ_)YwQN3qQCV7wqnWAt9>o5?XWfCM9g5>?K zyox@GPjX(;DCSY9)LmWTgdLH?Fkj3A>qm}ss+dm;N|mL|6r%az*1cIX*2tZPA8qk~ zNTC#;NEwjRFPia|jd8c@-*@snk*)5#&88icrb64KHm}uJmXq=?Yi8HBJ{K}GnMJ@o zxk@=k6?`C4k5QOu%77Q|AQS@=b{x1lv``l^Hhj1_Lp9ixO-JzXZK^We9HHZ}Ib43N z&7%!cvI%5!Wj14zMdw}4lbPcVDU?yY5n?|ZCf5%m5r)(B0{u)I=BNFH^?9lfV(CkUSvb%qI9riuSgfdi&KQ;6yGgJKLPt zl+RQromQ>nL4X_fE+-ax*m6tq3$*5oS)6dX-XzOcCWoE~!H(cj*lv|UmmPnTrS(iZO2$1d0t}fgMxQU;oMLXSKVNIkyGZg*!ca zaI*VcP9EbwUxb3O??!$8iWSMxnbNhf zzlF2mN?*WtzK>>V%wIU@iREMV-lxIbauSv9w!LobZv3lEYo@0+lNG?cTrfWhT$kGY z6l|Vbkl?n+ZXHl8oV$@;gt#Me0!Wxy?1=(P=DIKTnmJic9zqMCne1Tso^Z;^LC2;D zGJl!91JY(7;^~{E7V{<@Cg7)>P)$Y4q=TBApZ=m(dD2k2lDvD~WYF18C3^z~cp3a%Dw$bgR9-XOR^8Slg096m_z2cOV^BBb~vvLGR+3X#Thb<=$7 zCZe>?=|@j3QA6pgrwGNFtJ>d+Ir1&yOsJ28{25~ZwG{4uWx|?;U}?upY%KgPoSZ!@>3O?+Qnf(g z44Jk+m8=HzV0+SAn>sqoD7WSJ6O~f7iutTnI5q}YSGEO{v}a3mbx!S#w%m4Eub;CM=^2$TJa;{u9p zRO6RafStL$#$cF>^9H%oL`^SGmL z?D;We_5TZd0fp-6}O5 zn?s2<*I-AI0!pe%)ZRzak<->vlk$g1xB9s)U+7wYbR7oK4>DDXpz%&zQiI2w>&7+r z_&%D8rv5+n-a0C)uG<@?r4cEm6{P#3K~kiW@X9S0$#0P|TZFbUNGxB9?W< zRX!Kzyr)LfEHAFVmqp0bhyD^hsZ>I?47Y|;(Xa9y$DFBeFNg6&gDYx< zI;s@;PotLuB&Vc?yIl--zCMB&Y$jQm$&JQpPMs18s+Py^zarOek*jc!Pef@Hkg?mW z=pCt5zfbPzb1`@!#XBDEA*9^-5`M+sk$A6?AdRJ3=+k(zwXGCM#=U;wY3|u4+$N41 zPn)}+z4@AIi~z^ZZ@5;Q=D)`)=QH@@`u_i!Ii>E%ieOY9J!M{k@Q4O zobZA)YG1QSXJQ$8mRD3rn}kPh@DhYq?RTvVT)Ls(vwgbt#bl2ZvX~RA+F3%EoZY)} zXX>lqK}Vek?kYT&$($@tsJYmf5Zj;Ss{y}^PX66o%ao zVlb=D`ShqCMej52^bGpyk!bCf3gDDdCkmI2^V%!QEP7suwQ=q%gjZ_M&m)O+wdoX7 z=G896*7nFqmN^n9nl-r#AGW3WzR9jwedV5I)rRvgW=KxWR_ zCW9qPJw=71UyBVp5YSO==)O0+VF@Y?*+-+$_}2nm0`)J^pX_4 zlbv)~y!8#!bcZeeOWDmkRg%mthCD?-zhf~)GgM+pW6>Y$*#DYT{55fAa%RwnS{q13|#ubz!hMI}trmqFer|urtX5kKEqku1(t6 z`)uJ9^YU!BF%DNjQ4KfUQH?THE4tBJPb;G(1p==IZfi7=hh1N-6k8os6s*j|O=ik( z42Y|kdb7JcGLP&UM^Tm6)K>}}E`OoKJ&(Ry%j2rCp_R-DK^wMdkHt_OrOesRSp!bj zt+Jy9`WE9SgI5WEEt}KmCpI?sCEXoHNS*!8Lzu3UUKE~vp6FH=pS@?Z%9;9Of9yELpqQ$}zUMNoBy!7Z&`qAziTXrCLmT1CMC4R8)y98rA2^9fQ{s%sIE$^z;;H zr?*q=Nw>Qi=FMWUotk?;{G!Ia_4rGYLUI{t>>bl3;zrI+1La2#4DdOkSlLZhWP61u zT_rwfdMHSHdZ$);uezx` zKsSMjN%Wfs+5rh|)_PRF5-}~d-_N>3;yx<4?wLG7GUt+hR&}~!+_$Fiu2QhW?GayU1%OEw=!}%G0>Y4wsKaW^WU$sKM zGvKGdpv%nu91B_YqH6Sele3nF30pD_%R?pWqCLDQ)@%Y%;DfhShG zdXg5{9t^dbE7(}&}d%ZIsBWjL868DmS9i)Cjc zlQi-a2Nla(pLmz(bP-K4GjvIXOILZ8WyT_hK5X`-Z;;qsxmG-Te{mWRlj}gv#XoNp z<6};tXKj(^QEH@sv1@w24*%1^5alml`gtp&W%9El@56qK<&EHUb0YS64a)^Zu6?SC z*3LWBZ1P;V1hX%MoAoYoD|af<3-T)DPNGWH4ny|RX2O%~-c=O&$9G*V>NFiDV5U9E z=o0u~&vq?<{L!z{XBIJ`qL4MpE5tl=%lNH%qKmi_ep(FupnSw_HK$vvIEnGmz`x(= zdlJ=oFUZB(hnGTpo7nS=&hX2hYsvNKoi%dgIeuSvauDd6xAy(vB|TyEW`D)*)k`BU z25Ccn!J}k>rLnevg*1Wvr$REU7_MRB<$J^yV@$I@k@d8ud8g;m)lDNF7z)XruPNzU zuC|%Bl8$C(e)U|PqDWAqJ*q5~d+5VGU#OPQDe1xgVB5EDGexlWkZl;3ZbPy=JB;#$ zep2CQ8{B@|w?tPOr02J4@2&d9SX&j8N`gvxdZROY~U66d?9#G z?@&!=@G>ezuw8M(;a#Ndu*>8_x96todLvJ8CUtj&8q!}lvPrd)pjl0(WxTXU-z+!D zqHWlIE#J@7nl8I5ZAd z^>=KQg)vDw#%B z-iK&1NIW%DPCwq<4{C9vhmc9!Ps|PAFWQu2Qah37E0JyYe_HNic+nUgHd=IX)W+J8 zSRm6!VS}kC>}4B9lf0g>^*FEnBpf2Mb$>IzidKGP!~FG949N> zCfLIBt)RcGsMD3Ew=PIgSH_Yi**nCGP!n0%9DaADOII60INm_nGfsJuzt#IZv(%kK z{Z%CWmeDi1j&c3)yZ9oTQ4@E6Sq1FU5tMlo1~XBlgvN!-VFeKmV4o0KrrA3ZKHITG z=xUlPA{5(mTIgM8WJIoxQ7PYz(*5+aRjS29>buZ<+^%O430I2U)K*pS(j4vbz{S8Z z{!zS?fS9LIZGjVW@X4o2MSA#cv>`@eGg71Psc+GCXu0vKN+Tc`AB)DBF~6;{3aUeQ7JP^+*3BI3}VlDQLboVV5nXbBLtsJu6=vCrsBnpTqy)Mwess)J5#Msi#Ua*+~WWu zit87jUEI#uYY?;Pm;0Ls4_jYtCA(^xc`u1nB#uX(FR=LRHk=0jdg;R)j+q>KeGt-Q zaDQ_$G2)kik%Yi;Y?CyGAjQZB$h+>8TVrUNvQqI6Wvft6&JIv1CVXoae{3#GU_Zc< zTQIAwT0M6y3995@)68diImwD&9yL~-#eU6uZ+@ZetYP-koyno|lUS6CT)D)(`LiqP zRGydo%qVG)_^syh+bT+fpB>6NT)!hY7U!k(Mt?}K{M?A5Hp}u9#f^EJd;eO^%QZBg ziMr)=+w`@w&GzxD%{DymWMY{r39{3{cdv|74Js+pw)I$oME9PiSA?6I?%5LWgvo@S z%a7*WFQW?A3xZ1%Bv%g>un2;@6= zlcuMZ(tMGO1c)dV=?FB^1ZCwCM#ZzRw7L0&J~KyceLvYCm(=L2^IsRPAsE*&?U4^1 z%`8Lj6w*We)<1YD{CGL3>fauP@&0cgg+Vy}9DV`cZRP+zu>J2o+swz!^S7h0_o`@W zc-n*nxPf#C=$K(&>SnsAlcW$HlbRCYn^+UVzn5ajlPZi*Cm>|6|5z_R7fr`sC@PXe z7OY{T`!wW<1&V~WB(f{1L>_z_7IN2H#%USx<4nlLz@}=!*0}$5%lV%FcfSli(HuE< zT=LM9FL%4vDk$N@ekOiI3cd}`_7DvX?)GEJ`(cL0cp)krCKo~-G^F>X@Vk*(#k}sI z%8jtIroO>LaKL==!0BTa4(|{`7;EZx%=+8izwmC;T3|9HehkBWF*@(`!f-XbUlom0 zUkpv;wnAZp)IJ{yZ&}&x0BJa5k-%O8M~=iq$_KG8@l}WEb>HjYXa} zTz?wrUX`>#Mq!N)NJ(`mH$Sv9xTm#k| zx`u#3!ZF-R{`NP**e+$PLH9eQP)c3!%k5Ct5FGtz1M6TGeP z7*0Ttnv<@t-!T(Q=((WaK7Q;c7dLo0mK2OkJ-q;oONQ=Irdk+l2fWzv;#$b}HlkyF?hE$YICn-VhQ1H^Wpa@0hf2Ni zyuL+J<%`cT)6=sfJcwQxiY&JO_%6wkI0Y|m(%b0Z;*nKH+@9{D`%=wfD0rT0M@#U_!3i7~pp>s=ObfIIje~6^c{QCVIN412Gnpy~kPhW@o@x7}(^RB5+o)Fd>RoeemrHAv^J&{` z?oua`b>|*kod;xEoUn#em`|P>=cMN-ivTndTW2VoD%TPZSlr)wu&Cx z=~^hDBHELML`lt+s7PBegbE(q;t#Ev4ZU`a+3+$*(BCx;mMKuTNn9`S-<+Fb*BZU- z-gtBJWLbUD`vdSfc5X8MTEVnl2GWv7E(hd=jN6k-*Dt!&jmFk-AwKzn#ZoR~Bk#|} zREp~g)WyZclOBGzERT49$KEme-I?M0WZTmT+bN0`*YLLQ%v(H_LISO0&XG7JZ*|!x zHYF04t4u`Y&l@Qs-<9TZLoO z5A@`W?pz$*QrR+W8CGs2^*Gm0o_pWPhTQEu?-!-M=8W3b|7iB&!^=gPd~e}iBm1It zSDXuPJ^f|*MI(Vy?R(~4LYppxq8zSu`Q|JG4>E{-`-QJAr zh!ygZ3XeEpV_}17Q;~n!M)h>|kmHiIdsaml?W+xK%cSJQ43SsQ9c#Htq~W(4$-_uU z73-b!v-!7tX#++M*7lNt{|>Xx{S;xuW;k8QMCZBsR<+@SEC;(&YntN^tC<{{N+4n*gkytlD`M{A&3CM? zZcj5K<^>bzOZ1(&&nHeZikc1%h7ddD{5EiKG_lR~r;;^~YlLfd*0CL;MiK1Pb)0E< z!W7z~uc{GJwnV#$O6G}_or5XF(T_JGQ*->kFIFFQ1V>}-okQ~NeBP>_No|`(tM3x= zEg>Hriylc}zjq%U;=htUt8#GN$;46-w{4J7cbqBvs1u8QEn!X-2 z74uX#HJwnf&Oj<#+>Q+h@9tJbcQOz&Ola>pR=YMkC}=n3J*QuOE%&up2YX#OhLl0D zTJ~7w)lp5bAMG>#WZLV&FCWrqJZ<}ID-z6OjYq6!X;J;gmv5zSMU(K92<*gp9H&Na zXE*U}%DcB@Q+`xD%Wr1Ps8BqA+IwC^QXJyGpvW#|pXrg)F!_m+U57_eSJywgNV;#p z!s6;4cRM27zuh5#0d!z5%ZLN9cz}pA(7+szK^lx71jOZ$0BJCM13*6kX)w+RK+A(P z7+M3MRY4jI69LeMAPv5S1JK4G4Q@jK+61I;V*Qu`H1ACi5K~KY3m5SEW^OZ(zM0$1 z(hb`5W_mMwS1@7_5X1&LqB%(4a9{z_Hyl`k^bH5rAbrDu4M^W`UaF-wg-OAbrDuv#FaYbhaBVoGm@T z+8aKc9Sn_u$vJ?~NYGJ$C&9sX!-$K8lPR?Bh8Y)kfHe-b8)m#rouHS$`T1)|9$@?S zAImE0Yz)R`;$!0m;=LF;Nd11?zy=Qt!UWbZbFgy&D{EtD?o0^*DIoTen3%l>5L1Pj z6Icjl2(X&$5H>DKb~XqYV+qCp!(!rf=S&3myvccw6Qc6wKKOd1=wMg zH}uf7G;smei46<`r3!?a(g4fFI+%SqW_DD0}yK< zbP^>e@N)pdFF_M8AY4H3C?L$#?>ev^0+ew90t;&o=3s*L`CsMGF@SNvF<^-s2-D36 z{Pe)N{+hUf{W)021B?&l<7OU^2Frn1RKME;If^`HLffj&w@#g~U3QhwPoj)uB-NEo=kei{QJ^%Cq#{@6H2LqiP zXbBDr_6EygbHZ|vSy&$)Aix*|2zUgo0~Qf%50-IpaRBkffO1YgXc<@z9ShK-u)eS| zAl@0Yj2HO*!4mW?{zU*_!T{wSy6Ui1yvbp=3(H|0pppRG%wKbGrC}2Cr`?T&z$629 z!~dFtEB&u!;5BSsSPqkdf0u&~=HHeBJuC;(|AC1d>^Ji& zf#o;z0{F-Q4Mzvypc+Wu%nM*82m8&u09JyAHS|dUHVE8?5920@*>^ z{l-TQj+^O!<0A*h4F>=|a&X*m0O(x~z{rR8vhZ}U03nG3h{6Od005JNe18KwP0TfQdjTzTY$efgHd~wa{`X%y9q#_I_X8z#Jg>&}$Ip z04@2OzJWPtI6vSff5RLQdkTfod0~^gw6>wklBl37BkPTtg@gLKWvB2$D2kglH*Om(qul7GRS^!x8FB&c2 zUHaPPs3&6YoQww$h zQ4h>v-9gsC2G9hQgCjtD{^%1}z6I3?TUQ zZ~C`4jsq5z8}P<)z{2(3Km*K4cw=qA{Dh#j0rL}rr~&g6f~W!W6N0D#^Am!`1I@EZ^K?KAgLx34%m-!ztIrpM+A)=m>HdCdknrA^IQW?}OT*|9zkn zRBr$H`~LCw{S6#IMgNb#@Be*&AJi}PN9loj@J7OL$5RD{4zQ&fIi0?={* zAtA7H6g80k53~diI-qR;w1mL?h5%YZV17dYEg>+!;f+5I0`naLXbFM&4u7L11m-^c zjg}CY2N6I^2+V^Bpd|$6K?GcJ5SRzi3Z!rF8bDD9%!3G^C`zYM=C}jeWQm#^ncPB;AiBYPiQud-U*ZkA!Q6=; zx&d1+C}{jfw;OjND_|EwU}p3ma}NSDrvIQD%%TR-4Q5e;=mxW>Vdw_4s(+&!%&vx^ z8_c!_(G6x>-`IB$n0@^l?O+x*h;}gR`UdT84D7$59k|Ya7XE({+QB@$|GdNT&mq=7 zhgg5y;4pVMEdTG}4gEJ>&VTiWg8R1rEWx~?|5^_k&0rar|HT^$1+g2Y_cvt-`n>*C z83K03fA&Hf0F3|13vKLdfUafc1gt9H{XF0g_y*(O&V?j2q{T%Q86TTk8EUwy8rnJk zuij?k1W@(Ab}$5mX7H|bDY=17+W*Z20U(=7`QtE6F1No z%-Pw1trJ+m4LFN`Hv%3N&>8re9N;Mg-lqcVfQ=0wFfli&;^GI%x^&e zg#acfP^kli_dmAyT3%6cSsI(+w(mA$To zNc@N<)e@1M`i{5fJ{tn(JKooeadFLQF+;yf8ZZp(hJBph|PWeg8nWY+!thB#Kgfi`!su88=DKM zaX8D@h@aO7E#cV?5$F)w@`?2oMBL!AHlK-fjSvjjm~EYXMd8;9Bn|j!PrnkapwB|d zXuY3JoH;Py&v4mGXbj(zmOjVtRX-a9myb;d$44kZ7!aS^oq>z=+Svsz6~0t{mkB;P z84l|;wuq{piaZ%*X@1GOHwwcC8d?j3y}cvr{`FzS))q1lcJfrw{$?R~cpWEHV0l6Va?^+!fwE}4d{<^~bvf7+7 zwACi!jZBqF{1)*5x_!H|frg7uMl>SEG~R{g&up(ZchK!s*47an930@L7auch8rwNj3ChHQF*00S&JP_jxBVZGIM9m!0^<3kI+t`BpsX5oac2(=QUDm zTI`i^H9`=)4cu!8+&4?R!Lx#yXF8|GX4l4kBth9R3AuP5N5_Wx_vq^p*QwuxJLGmn-Squ&E+hlE_iND!vXzN42rLoWi^|O-9aW>D3=^>%%^&rSebXyw0R<;^}SC6L>9L zM;MTnb)MbA!v#nsoSq)MV6%TifIZ~o)@a;532Cjq&JNio*aMx1gWqw5Oi% zCHekx*J3_!8r9d7esIO7*AMo1vKw+2=A5o<4b6>jRUiOgUWF@3_J5uUr%~K0$^uvX z>spLuk7sS&1na`PfrsRq{eEfA!qOKKlJjd3H*UD?C8wFCLf$K^7G=jXMaU~2Bo}x- zoeSobsObKsFS_L^gFZ3!wa|C1Z-hq#q(|%k1xX3n`Rn6kfO2elw zMR%SGc<&kWwzL{O%~QFnR#U#-;e|h3S!d8RZRo|4gXf0JEbl5B^z3`(>KplUiIjV0 z^62djt-Wo0zka>mX3rA_zHd@9ho-1zZ17)h4tv}sgxSA*>q`ULrN=!s&NzBR4C@^9pnlU zPdYc$HN+?P$1e5{SeKoX

!vV^cY;IjPYD&Qq&@h$S&T)*GS!@FY{%&d`ZA*z3n> z!txao@>PI#_2OM&j?(xJ`_59Gj&Ce6?(g@HFDh(a5SAA;y2@<5+!g|Ymw#ur+F(7r z=fwQ(OH}9vjqin9__G#Sx0oD8N1if|3SVCdo3Xl62)K*ArKbh+TTd zb0hbXXV%LCE$Hcoyxb##d*rOtc*@GfYjdC0*7g2y`WaEke9C1at?#NWN6F{(WZd^` zh%t9)u&BZ9-oaE}g>yWfyDPD^HC@wH$Au0XUm{uP6Mp8_mmU&zmO2Jly#|kg7>7@M zO>AHAex)x^POsN=TbZxQRvxpqvS4M9EVI~0+&H(iTspccU88ux!Q_s#d@E$oY5ZVc z=NpagiBEnV=nI!~WQpKK#mOB}1c?Eje5(YKmHuQW>Y!Iot; zW=<}O1>R9lMuK!tVsZ9vuC{JQTnZ)8A##)@TI2)Ka?-0amC1+&G3kNa@SSPJ0&mr- zndXC*Frv!dEuIPA?MG*Bg7@dV4JDK6y0oMYS~B`{XsFF}FEeW5`xSC9JZ56wPX6H8 zZG%h|yP%4W*-@v2CWm5H8d?enKP&n%l+x~tu7~#)m-u`WlTsC}WJH+$&7pr-Nn!iO zlcB1uUs2lPkH2<%;L~4JUJl3G8xP28Lx%7-dU|G{i?F9$`%(1!(-X`X(< zdWFy28mjLOlXnw*qO4iZJq&W9PtI@E5hTL(adk>#c$6j?Qlj`n{iAa6)aB!y{1qQ(wsl{d_c-w7OWpA_-|3!hOe&O4nv~-!CknW_ zj!zgynT_V>W(eaYQ=evOdk_|PEEBxmsxJ~syX#q3f9KJ8QuZT`Jl~B}4TWkkfAySn z%Q_dW=A96oswXektv(WpT#;>eAR);@&^XtB7hhpH+ zLgcl&GoGh3>RYzn@hwj;(&>4ssk`=kYtgfv+mV%4hWELMZ_vc*!U#vZ-vl=MNQa-7 z-=^hvzoBXLI!7Y)?!pTlXGnP!KLd}AV-%OMb0eJ*rE0)rAJ#haS1UoLMC|M3?q}#- z@5}|hu@dGH3>SZB^Vat~ibEiM${(maLFb}O>LI`h2YfHdZaJ&GIHu>>&+SvPQ*KU` z@PNs$4edBQV}wg4t82}=ss=5-d-%7%y1w=3Q!2Te$7JkeDJBarLP{6z&_)=T5SHBd3nFY z?Ymlxz}j{`U$No9dhyCh1maBe=pH&}plmBizaUu}jn3{|TCU*?#^D$6ffSF0Jlkc#2b9|Ls}3*F~TEa20JEm8|O9 z`{xf&C5VGxO0uZ>&~Nke;bne%~j9V3b)KP!zMBcE@bpv)oy zH#36b%wZoB&7&t<69O<4^tUVd}VkNEt)#AM8{$MZY5BrlJlFdz9tg^}3fr{1M$* zn!0x&{CvEOts@_N*`Xw3o0;kpBU~aGgAhBd7Rm2l>{_F?d*M-cp7A6cxaj|o-gf-x zHYmM@srz(h%#y(G0elpPsyylWFnixsRKFgBS6{Q1rQ`|OI=+kB=2CyvsB6m63dT3t zocjZ}%vLc<9y4hcqg4m@==+5xYdxV-6CPs7i+qHXVProf!cE2L`RZ+BiZu0+NtcM2 zC5^1OMMI^!+yzRcn+>Zawj6_9>ooF54vIsKEe4L;rmSx&&}iojjYal7v=!Yr6hDoz ze^2&9_5Ajb>V;_zQnReSE?TI+5!Emrs-^!uW@E!*y5OxR+a{BzIhWNz1~K1*4txe# zc=cv<8X|NmY42^p<9`{Pm=AwuCgC|{qg4`){Vw>2OCEBP^8|%^STpKzkQEy;mua6x zb)AwU|88I$TxIvtuw=H+&hz+f7F~@O!KvEl%=PVG<`mWFGMc$!c?j*9eUejE66%;L zCW1?*n<|Coo>UU-F`6Qid*mG=ZVN78RmMpI9(*cz757%je-JX)SpUwjQf=o|llpb+Jk2 z>{ZLpo2zxCw+A-JS3Pa;2~k@ck+ql=qJxdzPVz=77hM|#`lz2cOl0^Ssy!A^&9h4| zu64qd4z@Za$)8Os5i++rJ!{hECT#2|fA_NaT>8g!dX*o2s?fLFMwJ%hGtQMB7Qd>N)Og|dVLwx`B6L}O%^tgWF-hihSa|AQ82+HY>rIk^in42H}JT6dJM&!eb z+%N5`z1U1s7!^vn$cl2D=o*A7Ir<O+MZU!b?nd4moGY=d|RpmFBv1&6uJ1Y(v zGdxAul&6)`rfb&(i)h=PC_i|n+WNW`$s+k|kjxrB4HNM@(s!7OOX_GkQ&f>2To(Oe z8jCECMz^L{nvX1O{rvqBl>k$rN#UXVredOfjWd@iNkDymQU#WE6z#_;502^VOffl} z)F{{nLyGE@C`l2kO%_Y59D%IhtGfmQnr>*-2*t$@6v$(Doe} zyQ|{T4{UHUv=fSw9vl;eEVHR3_bSfx)>wwkmIx_sA@gq@?B~)_bZx2ZYtpk!Nwj^Y zdMV}d>KDlab;Rq`b@ht2@+-vdL*rEJ;pfbmjm=jk^l3qhkt(GNI=BPz*lqGbxv)xIOGgkF9hieq0Xh(lh0a(PZ(Na4L9$Z z@DGm(BTM&;I)BZZ3%6_OLm7jJw`RABmn!!gwic}oiC?aHq}I=|+6EuLALrVQBjvKW zXv?!_AwzV9NZIAAc2mY&{M1~x;lQ+M^(DKl9)p^MgM&eL;|hY;Q4W+Y2@L81ekVU*tkevx}a?HBC+k9lQ^UXlz6GPDV{O7MzAB8#)R@5Gf z%6xzKpmu(xu08OoxJlK^^vMA;t4D zHe%%JOA~Tcp&Ikw(Y~Sih8AIFFz)Mfd>xiHw&;YoO-a8c#a4Q`jz@G%+stB<_o-2J zSFPkzsEkBhSU!6J32IYqn{CVqb(Z_DDT46kUbUqHUSP0u zB4<5=PiP!cMVCIvZGec?O7>_or2TaGMcdo2f=*ja;WpVAx9(UwZCi{T&laZa-ddNM zI#fIHrzBU~BT_Hc+Q&TtbDt$qAPC-)$Gj&97%ek!UO)i z%gM@us(AWi8z=vCXlyC5@niZ#WW-l(Q|ZXGeHq(3lRMUx_>nt=hO@J)o2 zk)ro=*I38Ba(Oi=1CP^!ZQ6H7m*srm=Z{Ck`!U<^RCMc;W3f11Y`!e`F1d7WrBg&` z*?z9g$$szId$D%|MrC#(Ov3y5hmH^IQdPgI7F%6UQ?>je753pc7{s`;Y%;rh5gu$i z;4`S?89*07*b!D60Y`Le{QY}_@9IRt9ZRB`1tlt)p9*V|Z9Er=kDbEB(L|d`@VZNf z>rpJcp8fb~{Ut&jzlwoW=BqL5)&4lm{%oys%%bT%JE_kp_%qXJLml4fVRL=)Z^OG{ z6+H}|%Z^R1H2-?qo!ZbWf|NQhX7udYhuSvI@q>)bfr@s>HHXN^8|+Bz6_-)h>@JP1 zNg?F;-1kxS*^-3e<+q2aBYaAik7trPKaUUGnz%ciV9(#C8ujpgkb0+!Se4;)A1tNGGk>ixW@$^T1ZWp(bNE!_B8IYu?2A8Y zDQ@VePUF7yq2PFgSUQx|DH;BU)^LW-x@V0;y6Cr4!NI3Vg&t<*{Jfp~yrn8}t?R`b z&n5ZFoH-Q{d-eho8Q05cFfK?O{5Xl2pM7>#&m)?JE0tlR`%0y>_ss6@-DqDETHjIo409;d+@jmt0AyL!OWMa$$7a^iFG&Nols`Z-zf zkLbiPi(Kkq=83G>x{On`3-Kbds*1hmDGKTubKak_6`GZTB3ryJC(;PQk)IRYx2s8w zZ!117&BJ;Y!hCnHXHE(usz{NIzF`LM5XH)pIPpWDU)dlH}4;u%w4$+z<@iYK_%Nae)x%j=a}~J`Remj%Lsbe)@QX2DOvlFSlM6 zpM!mAY9-zaW8+YphJJMgQ>48CD@OQbZaD1V7`*M!)UI_}@tl_Sr8l+Q#E)78xazVk zWxJ(qN+ajW3>(c&FM3);DX%kg+WdQ?^`${)X)6G--_k1pFhN_uQZYw-Tb zV&eO7T+*#l_Mw1GylB(MJqg(NX3Kuk%@ggWIaF4r32BYWW`{Z2g_0r;M zOK@1wj?lL$O`vGLO8R5nCU*llh=UXX5Jv z>i&mtMMNp9yuDO4_zhk3EDqB5oJ(;Hrxx7rQ$n1LKW@MKGUNR&A`wnJXutg}Nw58-{n65p&p- z=mYM+H;NwYL*mRA-k*=?tyNRLFooBw^6j}eP{`x5z|*nJiRKym4)t)J8VZ?72fk32 z4@0>p=d2%TS=)5pl0zlWFoGy*oj>e@SBDd&NSk_uKoD%qOa0U4Wy>4x1hMBuOe+n8 z4aImybrTQv?=PCVvtqKH@*cS2sICXGdx)ASQgR`Qs&VtU^V%fO5`G$a2FVE4vM^FR zcBG~=)NAlS(Q~W0h2B@{IN-A%ryfvDyh3+^7nZE&j(zy(v2=HZUc?gb)}T}7B+rUk zR*nF}3*!6@QB5l#DiDSWd%*1)YTg&e0wjaOk8{)+nBKfk#)PLAmqNV=S6ehsLb2-G zRplb1UF;5&Z-vv0!aXU^;`;vJwpPCQ0^o}Q8F+dbk& zV}XB@_-Suo0@E<9sTc<4ocOll*x zFQISKCOwr_pxMJHL4P15U?rdO?IK~yC1Pj2Vz=|8QEF!7RBg;-OSDzrrAety?1Gn; zsNy&!)qdaZlF`s8D zW;Cf@^bvAgHTWHmdoo7!DBbB6dgvi{Tt=s7rMdPaY|WR2WYwzjS-5jx6qdMqv_Xd0 zZ7JuXoD!N>QL$xuxaVJ%1(J$cU-~rBi1soosF~2wDHP zd4{+UD{YQ%UNZH9aTV0N$+Wz_bIEV$eU+z_4G@l*rgCOK640_bwya2oiza_~t6~yU z7o=y6LNc@#Bu4UJb3D{1#UrEZ)2}lA!~)~mWWQ(6<&KJuoSYNI=)P>(KF2US$jW)4mHj*C< zBgu#v=rhl1-pX!eMr9opU7aAzAx3t&@0R+5OXg7)gSUyC+ohsjm^)L#e*IK)+(?n{ zMWhRo2%VkuMakQCuRS8#r1)QX!S>O#;$AFO_2=is?^9o5Efy3jR|wb7`QLkH_(7+L zpB}fJkeNLmm?Yc^V+Wc%hBf>RbPrmA6$B(JqkC%Tlbz%4$DdZawqeC zo?X~#-hz%-Ns(M}C8iy5xeb4`8vKckTt?8U*2~QXr;9OJvE+R1IiDAz?hX>8ZV7Y} zKb>N6Y`fLC-c-6*{{i%UB59CADE5kLoFY^@cTRtgPdrBiaej>g@ zk9{7GP)r`^ZtOGEnt#ksCs}354cS;9-Gv}{-wnaT-DO^XTY%-_O^2;I^dW4ERahxE zNHu9aZ@(e?vv1=5kJ-+=>9vm&u^%|MO~wkR-&=`r!LQiNlW*;+V%^VU=N^x)oNZJ7 z;@5TSeTLe-3s+f#y_GCE|INwt2Zb|{B$TiEDP^O!k7k22*&FU$>2jXk8l;swt{tMJ zua5Q(?FnZzE5AgvneCy;dG1rxHFrA^M?sI^iyPG^ne;HoszF&4VJ?ElSog&+&nuN5 zY7}{aeG_RtjvT~!tj<`2rM4C*x3my>&*whtc;GHnhZNXkS#BU!Z+ae+L$YaGrmqGn zA7i0d9%z-|B1HZ2y7V!hNRf3Kc!GE;a40&so8sD!-M|!xe~(AM!X-7w{YUw$4%O0% zTRZGJ3;5%vOk6)|#^Kit6NcO-tz5L<#y*Ekpgz^eSZXT8j&uzu%~~dSkY- zFsE2wN^@b&Q{4Y>B}|BuN>_xtt*+9AHm%xaanp( zSBva*Gn&AJ!i6by!X1vD_#HDfA_i+AjJ%r7?TgsL^?f`7%PiRhgAZbxKYXm(7~3G` z453Nhz7dGZ)+hpI_5MG0l#U}@uJm(k3YkmkmdiK;=Z7krU!*|{5Flr@lomqedLsn8 z+$7HMR%H{Bj*vJBe$;5OyF!jXDWfs(<;l&~Zbg}D>P?3@&2+&XBYla@t6D&$62#*9 zHl2B|zx*@i3%88jY@6n#%bIEdeVOZYK`Ec*DenT*tx-+w;3M>$kfk&{#r^SThMMX% z+0D866j#xR;#VzwqV|8Yt}T9Ioc?X zZ@oTctqM#L_l_#%5xYknt4_>&)@*CTa(hDTQ}nZmH&vt-N2WASjlO5KzUV)Hav&0O z3yu(kx0TOx>~MuA(};ZR~&YVc)pU#8ml##hQ+n?ec4VFa3Hu zORkGIX($D7Ry)*<)rI0tY|I{~vh}>zD&MBuZ=a!}6rH2jy>#lw&PT*;^-MSSm z`$ymA3}h!QmcpPsUIK!i?xZE>oL!REPd@?`%u=v-=Ui~TdnM1*v={fwyadZy?#(e{ zuu&)=J$W@?-$Zt&`dcARh~(JAV2)09)Zfc)Uy_2%Z29oJ%T4;{HzJFHH zBK4JiwO^zj-AMZNyS{~dx3bXuuhF=S^wFV%ex{cV$bIf30^~e}t)=(fwGP*jmmZ`b z{Bpu&Rl%}-a=zsoX;fU`W-i}jiK6_T#n#s%%FaG3W9qJn%x&P5{`o-8GT}vBkuu|> zcZ+z)^!ZOkHK%HV5xG47AI9D(Mzk(kw=LVYZQHhO+qP}nwyRdzwr#JnjkErJc5ZT$ z`*2@o=J>|TOy&cJF9eR=<@>@^%mG6=yQRUwc6Cr<`_<$tMwJZ16!IftY1mVir+lT^pzSWDXoim@k^sz(I?Ens45eEFOf*U!f>mb?w>I!*b=j7iQ1XrxKlo#!saSh~0=eN?t{y+rUF zzR;XKQH9+K*a63y=O{r6aaM%vh>zJi`VdEW$Df*ziFs^>l$#=F^t}=z$a1!M73=ic zR42=2Z!G+E@bZXFG0pDE@+`GV)w7S$b_8~DqM{E&91s>6qb{TGrL!Sjg(jU7U(FAe zs5Zv|5B6PD$JcPD3Os^X%&eCsG=AEyVQagSEmbQPhB$Sz2x~8^nsswztc-c{Ji5r# zX|!rABAX84!!wuRmx4_qj&;}_VDGGf_%(2CTpfC7Q;nRr)I>uqZokuG`t$|Qeg|gu zO4{iiwSz%hc^qyZu0tbxUN9w&|99Vr?H)ON5ZHteB>$1sRwHLk-3YLnrO=EE+cCQ7 zc6M`eTAWV1>?Q`Kn2d$LnhH9ZrDU}i%o)y|b&kn@aHKB)HUvxIXEwwe7q!apnbSr0 zIw^ga7-|n9s{>|>iRl}05)>1XPG=e}G(=rzkxb5RtBK|XCkvE1^g72(8ZG2<&U3yc z4%`j<=>tKy>1B~Ky#vgM{R(Z?uEM**nk6uEO#~Uc7|l+yB2;|7+{MQL27Gwyu3WEo z`JVt)))nhEAIHP5yuKtcisa4aSTL9FX_^!=jB|~inP~Ulb;*}O46v;0465s<-%jlu zc{3O{_BBNA$^I8f$bf*cDkT`K-3oG)hP9|Ce%af*M#yDgzFMN`a#7&(&`*2*3$zoJ zRknm9lw$aij*!|W^Bpi17jrTBE=QwNfgQ-Lj1!*yeArgq?hr`@AR&Hg zJTvE<9lEgnvxk$^nC+d@(ok~~4Ou9-fxM-#QWW;vcvRWr`y`1bAo;_k0Kh8od)cu) zwK=U`4;m<^FqJxE(M3ZwOvY8;Th^X-`+8RQVQx5iU-c-lFtF=CF-o=?BJLO&z+LBH z9p5uIdE4(jYa}_69*$|n!8PM%2P`10Cuv`CNJuQs?6-#R1H|dFv#QcN_7ILci5H9H zHEHXAvt+C=dGKGBUx2RlFB`nN>K~ie;gtRPYTIhIVG`CR*(OaBOBuZMaL0%Ev5V4nHG^x7ZlQqxE(OJK zeza^^wtbfvACZ!a4*1QKlJK(=KM@y*u4!y`B!EJu%np)|_?loSR-J(CPcMmJ^eKPB z_$dW1XWs*;W29V}#J8HgjV`J|mCLCZ9m~VP4$J1F$xjI_$&~+=J#57%L_%WpN2gTTlu#!m>x(HQI&WEcq#k^`@&ZW+QQgC{Clg*pl<4gX`Z+ zmr?pNb4=;b!$QP_ZW#P~a3Ma~$Zlb~b-HT3B9P%FEQ_)-`(#4muXS^=9P>@12H&Sh zY4<%voH=HerJhe4j43%!-g@)ea33hL+>3K@G)M`jG@EqP33a!TqrUsEpaq>0`eipB zJW*Nl>y)qQ)AG=kW-}KosZM%*p30`8=UpZ>s-$1qau7V_PPuP0j6Kb>WJO-XYI|n~P5i*sLTyP=v6#*^Le6$nHtIl0A#6NYo!s zwb-l$k8|Nq5~BVJp`XHDG?)DVZG>$Eab*E+?Fv7R;o5r$XI3b5l$>Gr!Ils3>xMO|oHC)(04e ztnC3l+`;}dwvX=Y(IlbPmtMNcerb(lbGCzjMU7k1VPQj4S4Wry_u!mlT>CnZP^M%0 zvJhw`i#nA-+EeUm^_%o6vIe=Bn{xeI52eIv&O(KL;rT+7gs_mC86v<&+^rYRBXfSEbN`FB2OukA4k2@}{0t=|4g##|h7G3p$&r{dKyI9j^pvU>_H%qj1% z)uqp>Y%SA)g>N%^cdo{MZs6Xp#n+>jx6w^v+tx0GbwGUY6Rv(<>I9wP>&Hsh7#uT{ ziIICs%EN8Q9oSgqrJA!YNS&K|d{e=$k=6KX)bYNI`Tk4#v#vS?{UcYg(9=P)LartD zKZoM?1aa1(?)1IqcULG8 zxpHM>s_dKxl)eWgs24YjmPu5#Trweha;BsFv38j;8!|B``%R?@3wI^~8>OobBrv9# zAtQ$j%S}tATx*duDglfb_zN0Mv{5EXFRuDWFYDhC1BVd#CJ}^;+o!>QaEHq$@2JW3 z>7BQ&avr>hR6MXE!G~a8%407s@X}>6mU^)e;)v%XteSK!Y=TV3i_+MIEc5D(KuY>i z?$)DI0yk)C9-}XspBe+9=zcrQ-{b-Pc`*Knh=omovTw4C-(HyIt5HbLPdAjTF&c#S zp`sfVH{-x|hJNC|g^fn_CV+tX__Ccp&51E`mh49MS<*9cG;k>(pc(q+;PCSrb@cFC zmkoy*%A}`g9Bs3pXIme0HcbDJ95-`Qo|pV3ZU3^Y6wY%lt3<{l<}us>5tU<2UOo9< zF@8l2^p18y?ciDfS0)YfBVBSM6r3@@%UqJpra(cF@MAjP$z0O$9Q7{j!ptu)8p2mM zihKut_!b?YyZ9qBR@yW=_k=j>I4w^Dbs>^iA&cra|J?rP0Ql#S{apsx<*}_TjKEGK zR-w8!I>1+VxH5+Zc_qGWI})sEwR`1f@rR0=9HNS@n^~79{-IrxM!u?fF2P)Y0==9`SI7p&l4e>shBl`$wJ#RRKTktIW-7>GE@kVeo3L%EcenOYBRwY5u4mR z%N~ig+=3qsbptmOWO9B;9YvCiO}}m^;kZ8T#aK7@UhmiLt{}N~rOB{CQNinp{G&%) zD>qCyX_Xv~F|(ldpO-aT$clM-E+?BqZ|4C4$kVX^_)@=6LTSXI7Y@CClT%d49O#Vs zPUdR{$x@T(K>XmQ3dRb-lnI!#eZLjuHeWqdVmB2u?`<^Q@mYA(jD~%|!;Vj&y=S}> zBheg$*UbKSw_G9-*LaAk2b+*gsP~DoxCt0S;=J`<5&8fs5ORFbQm^qTu|H>dFX_^! zY7fj82g}XUX@TBTQe0HekL6hBNGvh^09Z?wOLn!ds2pF_^N+v=;w>f9idY@eeO@Ed zOPxk*4RNLp(f)b(8|`DLl>@51ksxDkFkp*h+iLU1cV)@wSwC^4+MG4#JOV!d*J=j{ z@s1-8G0-YlJ?7v~{>;!m2K@Wyjd`LTd(aqLI}*wE=(z=4@R^?M2)7HT^&loo03A@LMLkvT!lHa@k0K=Hn8+o2V`RA=!)4qSho03- zp^q3QP(yes=5@@wyn)>Fe&Gb2P(0HGLz#ZQml_G?)6;LH9eWyp#SxZ@ zv%9g-cei=`&g@Rg$V8pD;Q;Z6h`j2zwzR?*)KB@P3S0hQOu7|)zGkSsxX#JY>9Ah| zT_+Y5wipp|=0HRTo-s!_WzhlV8}5T;41Rg93~)Oy(6V>&kBk_y0fxei5t|ajSM9(R z`uuxAwk{P*pQlZGaWtS!u)Wv|Z@jexms?LuJRD@(5BScws!g1Ls+N~10}xY_g5J(G z(*seZAO zv-KXgMgyH7Co1CyKAa*!zN{<-UFMF6B({%b`T6zVCRldF4p&QST|`@+X4ms>E@L#$ zp#2OrkKmcZKXMtzfn>mPwu;}G#<<0%W_OUL?0JI|=$&*M1})_~LZzM?hPc%kkGulLu^OM(*jN-m5WIb8^_2^wX!G$P|QezSL> zc;+|RCM0YaD55an;4^ghV!K8teKbVZ4>t}5Nz&ewp~~w;mEFuAB(CPbY4=r$9)X8; zI2igo9xb}x7rY4O(nuc3Yz3`daSUp?V>>=0B$#FA%J4|NL#2Z^e%s@=7qpw)gfPia z@WYm;{0-4nuQFPS9#53mL1}(Sd%jekj0w%0T1vQi_s@!&^nQ7vd`!o9a(4*M^4;A~ z=XpiE4_9n%DkvVIiX#$p2VTz{ogV(lB=k7mb@U;=TTF8Y(rfU9hvRUXal3jtjO__o z&P}m=9A0jou+E+!bmVch&*oKx8W{X*vJM^+S9t;h;L%@KFmU?2NV3`jp(|)xk+Nne z3t-q{c+pm(T&j*%r3PcB`2ZMcd<#1x??|Um)$FiKnZ*HimU?dWJbuh-A)sb|4EwcrSqI8yunNXo z_jyrGO7eOZ)2NeAr}1;_m}y!T=Fjb*=#JH-=QJ2qP6$Yj2Rdf9_BZS;m!uw!C0-_e9JY4?KQ zZk`kt6vwpx%#ModkEwls=rTnKiid!WjMUNR@#@dZ;WVDM;D>S;HD|Yn0vVTQwGSEP zhe>X`=&22XwHupEKeEC3RqBt=ka(zHo3Y{yG_{;R@afE$=9oy9Sldc@zFLY0-fC6!Le4S9-L&!{h>_V*>F#YD|*<$Z>*NF=o)k@Ek-Q41$1 z4XN*N=hA$X-$ZytcjqWER+4T{h!>f=6`k7yQ5`$q!wxxA?e0v|aLAfwN${vo2vv`i zd>}U}prlM6)K!yNfAS=Yo?FjJw~AS3WHMw!O7##R1-RSVVlZP`Q7+<{?&(z zMJYd|`5Tj{t?|o;uXL&nrN_U$9vMQvM@59b=f!iuzRQDR?<}onNk7Hy=x$TJ9-2%#>9BSxZ)A(CE=@Y-l~1# zW<6T`NBfVWs@aVmlNh~MiE=xCcUZ2z?$tV?cOW@;kObZ8)C9ZKUiiUVSUa1gz@Jt> zTAXg1Cz)z4Obaq{nQLN!9*-GltpI1bm+Ysp*EE(F0)}MT@-tP2H>K3IIq+uXUNXyI zP7x7uoDrmkvH_{*PY>_Yh<1Ll&3+Dq^4S7ci4lQXomOGXBA#!6F9m3nsYgnY5WLa( znr@i!?o<{~}2c$Ba}PWZ&bM;eBK*sPCg=5L)1Z zIIAgV$L$&1k=^WL|s0rU#{&8vCfe}%G3=8i^$}}v>82IN3O%?XTeq!B+9yJDh8zqG3 z`fZHEGgDDKMWNVOr;pP%gfi^BEUT}W17kK9Kn9bG4PJGYm}YZJ8@Tlg?ycpv!|mPk zBv@Gn!9dB<NC>)qHxOcio_Mf+quC-`3i(h*rva~#{`0c{#N^T) zzO?Hv?OgWJoF>9-&bFCi9Z& zT9wX+#;x+iC4={fd#KGJ`}^%}gPSyazuNJT+b{&2h4S8Rx)6Wn@{$k{=#2n+_Ul~^ z$U4n;LH$uPg9O^uY=E6TmbWr)l>?@OQ+#|n|gM8MVEA`(%B zMzRyrxPlzrHm@-IP+MuUvaunK+w=WsPo#<8OE`7JFysDsU%1(AXK!6=Muet zHp|P^?)@hFk`m6Q?j$%do*8DI%(=jIUF-E1I}GAwJM&nA676r4u5`!l8M$GW zF3x%Bfb{QVezl;hHdhKRQAd3#fz+Ms#XgAjdhW>AEF<8%vnxDTU*?X*qo0szVHt9U zPsZvzKONS`hz-_IJ=ZpSt}rvEoD%xk_g;9&BM_idMX2Y=n|qoH z`&x6SsYj2L45~v_aCY7K99

wl7`+u+f&QIqhGDNkiv&pU-n!d&;pIGc6+}^01`J zyY-)lwJp{C{wSTTxmLQydq-w`+>v~$O@B;)G%q(gk z`7NZCLt?EucBUffbHoqhgb`8NxUt3NxkbOW^fjXL_g@{(Ua06*P%Zt2pr){`?}_B3 zWe0Cp?As$QKM;m!rtqUp;4_ybMXL~L7GZu5L=!w1zRRQJc}8B2>#13h6$9csWGg_f zYsn<_49Z@|Grth85W$#&`g5y>%Nxv(quR!EWS1eaN4USR$V+sCKH6NI?;e$l!SDMA zjkD(FyPbaDdKi^}KrPOQwC|c6HC>88{~eA-sUVE1#L0OCV1_aMr`J@FIQW6f2z!nb z*r?ZjT_nGm7&Pw27~1kGUH6$IJldV&rJxLw+rq(;!+BURQsiv1ZL{o~IfnD`tBXfN zrGJt)ws}x$Ar#ph3~__pue6*gMM^jxIb~r(^zuGRb3s4w6Od;P{?T?;(+axk0FJmD z&-!5^jZCodeg2H@78nio>x#m$NWnJ3pmvMqaxs}@rzvra1(6aiM6YXvpnAaUhJgt5 z>c{6fTng;t-8nH41-;%dHE=zshnU2xZY$glA%=X!^nF%T6y!*v5n+|?f`fXk_-8$8 z!xdr!68c@Ih-4xW^z|0)VH`!`V@fQ6Pku`9{oHR#1bIYL$1p1*DHm@DLy^=Adc)1sY&+UA?UM@h*DQOwF| zHLp+4ltP_6bjU?epvB;T8(XP1xHb|YX=^9kyu>Fav@JxcazB6oc9hY%b}%I6po6i* zwkOT2j526c>yLphio=KA*zMmFu?teFn5DIn2!)wrOb6>!5Rl{;+MY8=7PQWIPmLwGk@wTVZmXBXx4_yV{}ox7I6d262bZs5sDqKj5b( zq1)Ql&o$yzO~WS@7p64iJJQ2InR44kaRDJb*Qd{V<#1W0+q&gZ3e+Za;+G5&WfMkd zR#Dr{uYNU1<)#edukCe8 zK-ZkbF7q~a)8Tcyj%rl{2pI0k)rNz5Fu%|7isVqUGjSV@!D zI6naSW&&B6;|Ckb%2GB2O~6E}lkeFL<|XvUx}hUxZ(E=On0{YCiDHJE6)wiW^(A`b zQ44K&lz3ZX^noAd+s2MuJAfo|7*Jilw4SV-i>3v199t(B4XuUVvJS8xJXiKJOkG4| ziwj@0u(+znU9$qtTgDtHC|f_YT2Hc4$|3o#$^*Z(JpEob<$o@!Q`0*sPh0EF(eDIKz%Ro zr8%6?It$f~U(>W*tMvBE;}K7@pU8~Xz$H_lJ{7W|DGf!H1OzIM1=tvyNY zIp&~l&d9}8JY*svLlhvn>tF&i=~?i#De{#o<)1qB#|Txf%6NNtXHP%JyzE~Hwdl8% z9n0SDaFvdaFs2_WmoHX;v!U{b3l9mgO3I>}&RI}Y^_f3JM~tvO2Z#A*_4D1)f9Ga# z^Dfcpl*ZJY^e6)2qzZunCe&+n9`n!+y~l0yxXcp*L=3{KQS;h#s*T~lkq z1{WMuWalr!vSut~?Eh2V8fq?T>J9+i4p&fjFo1r+=ecs#I{flQR*zL16#qE0?`4pRWpKu4j+Q%CP>%ME zayLN_;Q%Uu@uyk{&jvYo;C22j1w`B4v)^zJy|1U2zP(SnjEjw5LmrJgNJK<(%dMF3 z`V?DahBKDy^U?ZGT2r2lLjOuz5s-BxO>K+DfovYSxF^xQ;z5Ax>uPF{ zm6-MwarX}eX~ac0$kQ>D5U{&5OYRB78aTmHo!E2mUr$!`43;EOoZu`e`M z@(R7)liU@MT!BA_Q;MgK(PxTSm-v~m{b)P5Uun`Z1OXUUrCR6^P@55Udr$e_O~F?n zxm>2$(><4=ih@vm1DzXddBBrrvgRa7k_6}(s+)D&QBu(9nz)Bxg-{~RUAovlTZo48wIYq zc4A^`cTzMWr+i)HjXcFKyp(LA_VklYb~d<@-A$&??+voM|0Tn*ld10c>vNe;Gw;`6 z@&N@WI^0C_ulrJO8Mu0qAG=ZZ>J#In!(@E~IDeFeeT(9?eZ-h_k)M;RutA>ZJrzv|zcU2vlcDR|#6+ zK))BgM8HE%Ucp`X@z+v;nQMZoh5}lJm&mGeo6u0ryRl3;ZETcIQ1@!z1saC|o zvb8)k3Skw4WX<5A-Pq}hO-MG%HL#ldVxkS92Y~@)=1htB9BAvN$|$UcI2*V7dS!qW zsE@yx>5k&NG*N^)=(BnYO_RcC`UiiRF}-mtm3#eZ6YKiQor?w4yUxC`*pbNGYfl!v z+#(;SB_1oqhA<3U74vKkjQWB(apYKfAJOwA@5ze%Z)UOIMZIju<-bp8b)4Kbu(anz zZp6tVqOmaS>7IIS^s0QosDO2*FC8ZLpWImOI8<|duhGmB81g3J?6+Tcc9y+t8GTlc z^H<+BtBfHcwl_J8ZZ|qSKeIp`io8rF5w7+L)pj+f<^iAu+FAW8H$=b2@IXmEMX(~E z);=TyGB>mF#h8t7Uc>q8nKtw#RFECdTA*av%*6|j+^4N+U7tW5YtWI+`dVF)u7H4< zOhrK1o9lv$ApkCZ8g9K@=^vzMBy`7|@yI`J+==qbhKwPrhAo0T6k9HU2Il5R!R67*hY$&1x@2hJ@JxrF4W4~^-Ct6(LSBdZ|Kmim zaDB6zbN0F;?ld?ogV9Ro;4>8fZnWlb510KhObB{;QBRhR*guQ+m%egJDPvf+iJBV5_JCjfIZmnuQmS6bi5<9kTpUGCW!es$09gJ z{nk5CBAt}3ADsfEJlww~GK`+eBVd>Of_6ZJe2cjUe{E>A_#mp5E4xOHgj4sg2E__1 zt<2A^&j;G^OUvWVs4_f@T#hnfA9N$Yd}imieq5hv37Dqt{HQK^~E=Xb^;0N-k^EMMxM>hi@C;z^OI$dkJ5AC`{2+z2@ zJc8Bhh7aC~%7h=FX{+{4k1DA&$LKa@SB1*$dObi|Z#xZ2m1PIN_Kn|ZAyp|j^?I7R z#I6dn$A69ef;2eiwt4;pR2=9Zma7<873p-cjtm9_YDH^ zD+NhErC<6;vz#(tZOK|b=AY%!)YK*GHp#B1%3 zn8F5l2-px^0nM7ss=Mj2f?zJ*K#IEO8S=8$1|n7|%Do|CF9ywN6&9opP2;wqwTPSOd%M`U1TMV45AKY%?ilA7NJiMdN+p zb_(60N|7LFw=zqfrfD+;hdu;ZbOf`5c@%BfaBxpPk6hI{49#%)*x+(s3=-e9Ti?KE zB0_?|mw<=kGtyoP@^-Tn3!Kgo$+%c*t!4fU8Y&WPl8C3Rd7TPrub@tlOCGgCVN-ic zNboD6Q634kK5K+j(Qv3P3*M{%`U?z93CT>EO-2PowP7cE<5Z(U4cnTMbX2a-(^83j zSNEc0hU(kR7c#icbFOM!d3iNf)U20c#scG#MMu!e9;1VhIxy;*Kxt2e&A3;mk{e(G6Pr|oAm$$?hQUF-?+*`BR6p(( zQi}Kd4{~@-V0l?CJsB4dW%1SZv-e#c8P`PXmmE%>(MyO*B9yT)ur<|n62B->E!0Po zs+)DW?1M_jg4$Ut4atcHTw69v&kTHe*kOO9t^Bi62ZoO`ULvg$iZDnEOFhU%P@+$_ z)LmHzX-8DCW88b_w`;D8y3a3^7^zC;+>cg5)>I}`o<@N4$9wfPGT5uNzT%e7ilz=< zM*@P-Q<|AD%b%KG9`zXG9xcNNwFBp`n2moUY!REjtPlKcE$^`ae~8g}G1HMso|mvY zc=kwjcsLC>#}AzP03$r(<1g?_MGL@{;I!Hy&vWjc<+6lKcHVdy_oZ81>v#l_p*OAs zS&mL1U}$48X&k?#D&qxJIMj4Q%_WS^A#ZMwx&sDt z2T9wzxB~$Ifd>91XgWd;uOMe?yL`S$H;>;v@bD&OWr~)G} zFp{An4b9L5O8_PUa0KWPOE?A>pb9{NfmwQT1aJVnclZSt=TK&hpq)b71mM!v;0VAZ zqJs+)QzM&;P+whO=+INQQhm)Mnt%@i%-Z7c>e|)-^8+{pb5+Ao3T{sU7`1>;ac}_2 z38?e4K>HSe^B{kLT3AY!Spk?XsI;oEU}_QuvFzmJ;^=;dM1_QvrNb}@D9dUnK>;k8 zflXRiSp0vNK{y58w3&cYSP6gjE(L)6%c&(Tqbz%sQx6S3cYp(c_u-rzTYq}4{2hx! z8GyeE!6aK999w?P0RehYF|J6)M^6t92CV*`LYcX`G@7_JeTL_krM6P{4=s$K0eHFs zZUp`yj?N6=5RtD#J=FvCv4a*^*n&AZfPTv*xqYQC;SrINKpKJn(lbUN?C-H&|DeJ- zfCl~;V^SAf@^?g4RYd`K7qkLn90Ry81cL+x{ts1>pKY9dJm*FqL=XV-A60VCn&Nxw zcGO#z}fF6Z4K@;tsk}h&FOI2iNW<1SW@y=zC{G& zd;AO%2v`HKz$`#Qc_!-C?a5DO+MZe5o>>HpAD>*D8~}7dF7^1_CJ>0H;EN+@XE1;P zI(tI?++N(T$T*GLUn_b>n1Ro`loTXVBA*kTQ z_~v)|+@|X0CbCJL6)aoxhxmwp=1IT6_<1y^#%3Ua_d~qo<*Q{=a18D7qVf0VB&fl` z{>djjz6sHc4X!|49e`e-zYakJ)b9v7(YJjY0GlQwsVy&S*)KEGZ-^vk24|Kww+0Z0 zrw5<_I(h**6SoQ6zXbpvPC_CZxaSWu1pu2GxHtm|=>9R-3$O-9C&BNoj2<39_PYI2 z9|C=V>=*nYaD(KR;1>Yf1iu78KiP9%L_BUk**CZ&z`Dr~ft&=mKLUDy><#=OSi{7x zAOanfKSD_KAwI}K(G$2}JH>wy)o=70p^nNg0R(z0Kg6KuH++!8!dozb4$FU0?SIkd zf6@ED$mL&@?*1DMLZI{XLm+{72Oo5t`6HOX)BG#??+Q4x7}vz@$M(P=Jca2OTId7l zJ2L$4)7<0tYR-Sr}Htc6U_3|%I z#aA$EKV4iK0j{<4xbUVvp_D*FbZa>u_c7l9V54yF&v*8LKaIfS8QQCQTYGzPIXpDG z2o)M#gqM#FPG5Nb+AY7*%+K@q-5f2H6X`emk8dAXfIuDr^Kq%Ccr!a8+uvA2(;p@L z^2q}W5RG{1yAanCWPMsw*;Yhk3BNZ%NF+)q8>sOzS|^nhRF)@sSQmtnO59)X1V(G}(Q|o3cP*ZAJuoQO%CY)x563E8tS_fmE|&kP$nC*gQ(&hSfH$5e~F4@$>IOEk+$KbKu`9gDR=3Ty7I z7)s=k5&F>vn<2iFV=PM8mMJUPixRRwSSIIQNVl;Ca9F{j2=XvC6jzy3oxnD5wjD5K zyJ7JU2AGTLmF{I&eP^>+H;PlY)3D9=*B{eUwD(FVGAa6`VggJ6r`ZSdzXXUwLa2O= z2nxI@a(k5@9OXIaKKkUuJ1SyuBoDAejytoiKx|;EsYIKPMDtmAZAFIm2#(anz>k68 z(4Mz`x(#vU3jKbdkdo7DgT{0eI5*B zM~?g;mrTu6NXx-6rEu_lP1ch1^osoTZ`?}ZOR~J^8k*eYJ1=v$q!Kvf@0{L@-fM7m z1UZ=xug?W>J=!+dh849KD3;56>ZSK(EPfSKbStaBaI}oF%a7#(2_SU&TWcp#llK*o z5GrEHl}g-=0*|opp9@vE>(yb1v0hb$8>gb`R`1NaS|g5e4w@=YW_S<2 z*+&Ww$S)V=w}VCZ?8zJ^!_QJAc1Kb$S6g&0K?)bDvLPIrh@jG5BmL3k*U@rtP~I7k zHE*PdC*0W+UGtu5r+uUKK&_G?t!zz)>8;cF@!PJhQw+Ad)6`>WgVX_>pbs(6CGq@o zRwcT^PoicPbr+SMdrt5c*}Jt)9T{utQDbPfI1o*I!v@vjVWi(L4Y5)%!4?Xyt3VwY}61hx3&9)dJ)^6b(p(0g0H z&0gUw&`c0 zRjp^Bm+D>Z5Y4!MFn{VPq+wpav!)3%&HmN9r0A9%t?J)wlp$?w9&+LpLu#z`&Vjpt z?08MDqv0DlQjF9UQFH3{{A)j`A4m(7VlxRe?DHl=`)^#Z-pXgZ!pml)$IG{W?!04w#taDr z-RoYEXDTZ43f!ujZZqtQ{nG4u%>aWhDtw3gR`P=Vbn7(Khqx<6R>dJSg#Yc++~r&JDma(yC6 zo{Vuc6N*Op{QPHLEQTJ{qdO~9OHv+9`z}iK5%IafH;+*X4jy+aoHlobPw@%3hfnL8 z+0Wr?dylBEUF@o0Wkq|oU%h;A4HnUZJsT985_MOL=SOl?% zYLR#f!!zKktj$!?Pe^Q<1HrixS7wvzSrCsgphCO!ZuMku z?KVGu2C3E(E^41iLLlv^S268dJnE@9foGZ+mE+S{FCP6ce2E?@1!m*m^a+c0mbG>r zs(LV2O5PH@dt_3>(-;qcB1|eKR-pXVEj%LbnuH2lyZxI5JLpt`Oi-*U(ccVzs|g{` z%Fw@QF1ut(>4)&mi_(D}%Pr(9+>hOt? zI#2e!Sek0hRpPy08LZru7M6Y{hwbPrwj5&X?T6cu7!PYE)VFn(UIt~79Z;OVSKM!{FGq%*X+BAFoH=B zzuuc0tZH@R16Cj-L9cE{emr-e5=ABDGp6s?altQcTzl)vR~+d@InsEl8jnzf7MDR! zy9uf^K`Ci(tHk`2`;{TTCF;HFc$459EMQ@CeK4a0IA%d}R?KWMc5%(Ev}ha<8M zw=kfXu?3f-WXg+6r=GixlKf6GmA&uat8BUIDhOCEHICltt6ULx>-6m~iRY|V!j)pT zOBuNx$&%2;&hu;;FcNnq`m{u~Z!O+N1jClVMWb~4W@DyI4XMOXcycCS$DQVbc z;?xz0Jk8-Ae0RLYofD0y@6(2kf>tJ<&9FdJiv7%>=qRE#qK^2krzFE2BI5kLN&JH5 zzzwtf8hSUWbzZ*o3epK3m%F{dJyz%ZB`~yiNXzm5f|-N2(8bX=jl$Ph0@BO(y+3kc zS5xdN*Y29=X!&0GYVDVXYYG1f$$uf>C}#-5&&20TG=;i0>OQ3Ux&~hfq75Re`(ozE z@M<>pkM%484&~}v%a^)r*;;Pr=i6RakjhjNo$#s)F$!4_kS?3I6$K6=eyh=Ey z#o5dmF7yEz0#FMuda5CMGhefP?8WvV+KY1`mdw~|ex(O1sS_OA(MAQC^Ws1F2IpW>_!kh-Fl zuI5Hp%ze7XjmNsokI|l&d-=B0DDER%*AHjV2CB&llOmar_V(S57oGeg#fSG>S3dL;?wy}`y@fASd438dM4md69n(a|^k|-Sy^;La zI*%e1MFZOH(J$0Ja6zBYnUt-d+FFY}%JUj4@C5^6)n<}pwM?f#?MJ;v}bi5>k9?1+HV>|I{;>Osv@Kc26Z*&0$@{05BAzOs)n|rhi^T0Adew0<+f2Jm=1w=Q=21<%;Zxi*y!l}|KP4AzyQ5!GH^^5!_!b`pWVURB5IU#m zf;ba+{Hs!GyK05=I}b)?bQh;KKhoLKsoFIP={{C=5%1nO5%~Mmb35wQ06QXdMd4yL`GD zW7Q{Y16{YA_fQH`vPgXyX`7KZU%_|W$l6d6dTr25S#zJ}_DFv16P0K7x%;iuSf1N= zMZB>w7wy)v&i21K1p4)qO0Hx0!F}nC`z|6AAO#OqZ24eO;$3@CEa!#QFTHg0Y$oY! z$6pe0_*!S?^tFTwe~YrHf}cHse3wvqexo8Gb#tlp#?mL|;F)!CK_kXY3GY^OHqvRaq~|ESZoi>YaUFU!GunNuQUHsfK!bPaf9XGuhb6_pQE zr00b$o+J^w|GFjm)ft638Y-;>rTwJ#1-JZB}P&$oRSPGgM z+_1xs=B>Tq63Q7}Un$yeVWiL@*Qxf+OC4u4oiO5_=_5DNNdZcPg}QVw)j!hb^Y7{^ z`K!>!-B~|e@(xNfn?xDoWw(*9HLb2qq!a(D!&1f)>~YzRq>W(+T+5c`xCqOATi}s_ z@9@Yxa>Vi-b~{MVDi5~>xmU(k`UjESuh)7_IHwtPitwXhWck7S8qcDtcIGoTCAW?b z#RIh7jDV&wA7%T!_Tx~^%|?>Ij))&y^z@G)y3b6nKj&9CrdyfV+o;oWE(SvN*C+IL zfwd!7W3}*XgoEmrb+tAssj2Vpw1_lKLl`^`Z`_8J`ry=aU<(%&i_@~`HPotFrZpjt+niMrPAKTGIP}l5O>4f-m6fa zU3yi&mz%?X@W2zI67!4hV~E@->&?FWTl-AwG12hQhw!lGU8BS7;m`Bbq%1jvS4Y8$ zEk81wQM!KAAj+LoGMI|S>0@FcUW~6#Wgr5u&k}I7T^69640|xnt{bi&AC#vs!DBhw z%%9YMW%0$DR5mCjpv1G|e`=lA!J(;32@dnx;AVHbQZeh%Eyo}eKj`c4SjoDC$;7&B z5Jq6(lwWg~Rf`A@d^+%WuNu|5x`q>r%%sFaKm@Ch?{&;uEx9r{Dj-e_f*kRT?r-9BRtnT#f zu2o7!hM-|h1WX%QXT@IbvN#xgjj7}T-A#;x{8fW6;;A2A5|}CYiC%}U4#S)F`IZlv ze^b2t8+Ue}hj1zZv(ra$dCi^*ckt@n)vGFgw0S~rs{-A&UDyvrwS8Ep?Ff{}y5Q0m`$6e*g3h%{%Aa6feJXK-ldLJlTF?18oLxjO81Dh#5Xx0OK6fZ!J6mS-F{g#RrT?J%fg|v_A=VJR&B!G(G158TPy^%D6yjc^v$+iP_ zzOBw`$Y}I}MsG{B;*#+yYOMa{-jw{!pWk|fR>3i5v_y{cK!;k{uu{(Ld5l9< zrdq@b1{{LB)+hY>3DRq@R}NV2K3+#mTzyt;W8{6+#i8fa{1Y}<#I`5hG`-C{Xz&yy!L;r}1*}d;*E2v1;W;gfPW;y-*n)I0Lz|Cy&mW!fM%Z4dH2Kj> zgv?hWVm65SV}{mz!&LUgd#3jeOd9<(Q<~l3?ZeVovzbJFom#bqk2?OqgHkYfb)rH( z^8L0mZ}I%|pd;@Atgt!!Zh~U6(gr5Y)Zi+%CaIl_6Lme)uuBus&O)+JVZ`pQn={?8 z@o`H9HB_X+R+$`~=GP)IFgR9mpdRZ2qAGKw;kJo7nud(%7S%x1`xU2HJ{R`Sd|j;A zumS}wU*!b1_%$v9w^Q&A^$wIm8d^r4G}sXVUd_sIu_T0) ztDx0{Ld*%=rQ>UGE}K1){hhv2h+2OzL0Afak!w*Sl?lEeHYSM7`N1+w$yfi)Se(3K zM(AkaQNuv`mj^*fs&P)BD7gzxoX{Qe?SL`G)d5Y*;ni!#WzXaa&)JCF_->1Iqxpy2 zQ>jpXa_|~i^uf;`oKL2vQJPqk#3Wmb!M>a^HC>#{qb|0w9Gb+j_!SZSHePjd`I+_c zQAt-mu=37f%IOhx-3-QcfiK_rmauKVw8eO#{RmgO%i5UPOb!}L*m$7Xeed`c$T|_7 zyHc{S_%zPM66^2&x}?5^q54#2yxor6L|=nuO=<&?-X)bZr|-CvWgG}}D#_VV30%I~ zv0qy|<+ZrnAgr|0Lb6pI9uI^aUPBXPO;X6qwCjB5^Z0t*uAE~=j5h;KiQF*8`GeqY zmMb>2CO!CGkJu&*Z}(lRlJ<(oUP1L% z4O-jxmMpBp!LMp)TulqkTa1?8_B+t+tpKe%-TO~@W*s7gxlH$QiTEoWONtttvpH)i zW@>Ui^nShKT$5F}BB$FQvWu%h&O`e4YRe>n4L+a(Nk>0wu~N#Y8LJ?h#oE_xf(ht+ zh$axAk1HV$BUj_t%o10t~VqjXS$K*BjU1~^YEx2!n zlAkIuCowWWW4{YmM6oW6o8u)vWg|v6)Q6G2T~cYf`fT~9Wf=iJmi3}V{_`3vC?c?I zm>H=?p?1;xW9pd$NtB(_@!G!M;iQ`_4=eT03tQcCA1Eu2nz4Mxm30k%^4d^oL~oxs zPS{4|*d`vIKD4Bs^bSXS(e84ucLAMqa#2n-S(J%)^!}L7CJUtH=u^06Zx#Ebq z+raS%5v_%sGg3RPx(l+Tpa({^d37J^Ocj#iD7-B~r{Dp3GKK);yD_|%29tqiyD>eL z!J}~$2l-5>pSP(KRjWtGRvnv!KRvYB`2*~{WRy_^iR?5UZ7pt$xZD(fP%VnpE9*ji zD&PN}NGB8&@v(39^q!pEy)}iHfOG4qv*=*|e$1UitQEh^c5(yN;PD7zcbxE7e9Ov6G zllY$wa3hbMgiTc1ZUZRvmB^;BJ~_=5xaBG&4-s8#B9^<2eihqOl)0^ilPUbvg=hEj zV|raMN69$DRv$}oTa&;7Zh^;)OOOWsy;Kc|q7pi@6ZoCpa75{+vDY#C)B5A? z>RW_lm4k8l3BuSyZ76zNqf^Jp)XPzH_Vm%Ai3R$fc|UGt+*dykI&zlIg!U_0r6>C) zVdLvt+gSKY#rQ8vZ@fa2n%=vg#>MPzt0J*=DA9Sqg=!$NWLZlLoUE|dlD>0utPb~s z`$aM>`gKS5;-G(3F(aZ9O-WNrt zv7cUp)HvT-JLya0Koam$kG879HSjG-KKhKA%ha_t)l`)~MRDZuFX5<;-htE<*C_G( z^m|X6pKsR^IZ9HTrA-*D{51o-<5v3XX^x|d-l5HE8L%oeLZ?5}aw-H*Ps%&JHk9^X z?LyR)8VS_LSb#3eg#{oYpjZH>I<>@3JKh`Kc!7q-4_Bh4tWdJqvlpjRZO8zM*k;mw zXvLPiyY^g+dtcH#T5mCr#;nkL-%rIaGzP1Wj1?{z%9=~>@#b!a;}BO))hE(G^q%#> zLUz?Jbj!aE53Ua$y^f@e%&FO1zyP6}JMW0bNNOoNS5Wscv(8d=Qj)D9j}?0+Mc~d7 zRyn~#jh@aV#I%Ua7Djw0V@h<@@|ntC=xfuG#6ZB}e^fe~+ezaQW`s*a&95~K^+iwl za-2ZA-XLb8bc}I;@xe;(ImJ^TG^UdJrD5I;3Jye=-EAO6MuekTwI-SBakL60#zql& z`Qp%vKDc%Ch9ZxIZpW@$M;kf+sB@QYPw;6dyVWZIu&K!5x08(xEHhh6M0 z*FXV<^+uD8Me-PP<5Q+#YR&VEV5%7Fr!wu)&Ze!-@6=%_Zn4?NZNo=_r{WbQ*Hkx{ z+H0gGwbDPw-%3PoLH(f2(c`*5N=dVTD*XyrrC3MMI^DfNZW+N+A39}v!-Of2Q{dxl z5M^J4l)Iav{qBW(x00E`G`ZuHAnn39G^IuuN!>0lDACJ4y)tK-GgAI*&dFU>$2sTQ z-JSxGBF)c+1pVB)nojuCS@d7K450Ruo?09il#I5RnXZxl|E}e!Q)qEBl-hOG- z76ONzr%%#)pZLbtI`Wi-N##8)SkU3Kl0n1wAJ+Wr#_8exvKuoqyeyX5*l5FY@vxac zspI7O?S;XFf3(;+Ctae*rLLHwJV*L&to->QU?ACDg^FS;O-)9BwY^a#_tm8OIP9}2 zZ{AmI@)GR&ck!L$C&b9xOR(xn#lg9?>j+VN>tmS2Z%e=$E)1~TX?e+jsyJ8n!kp&< z^q8*)*jN9S;e`i?!ht3}K5 zTQXCa7VWm{9d8D*KaY69P>>lL4aYhQIblfB@8l@OFRKwxf{OM72sRxiIK+$qn4E8Z zH7iP^@*KI|?n-m;NAW)r2Noo+7`~fCVh;M=@HvP*au3kDwdCEVnu)h<%r4_1P{&j( zjwR_Q>0|+(7m?m@)q_u&x_IY4c{QpNHya3OWf^ICw$vXrGW?!mEiv5Gn78y#bnfm9haVK$s!*?YN1{!)rBEjLqx`_h_> zJaH)TK?{io^B6lAj6EG|EhDbvHa4>F8~L%ONyR9lxlu-=m_z*9g;|NFCPE5+DBmpG zzJVo%M=TLY%658#)Z8l%Y)S^&s9G1LLw_O+uRO$*I*jjHPc86s`9z(f-8HT_ArfH1 zhlMZNw)QKOa)XC1SqZd_AvW=m?n`7SUfs$45xLcW;%6MrqgvE!aD4uR z>!@#U)iI4Ulr*V~+(GKzS)vA%u5rHcYuhR~;dF4A?XmFNUS+tqS-j)NCZHOE#S$I#)}B&zP(-YJ}C#XlD%MZye2=uItDb@QepGy&d7rFLnH zF|M^iyCWoxRlY^3X{WJ_siK%(B*mfZRn49Uax;yiwJy5ZB>h>INQx{s)};>x=e1jJ z@N75y%e7u|3eyP~1wx8^mp(7Vyg}Ldg2YBliQ%(`H2}5GQFRm=oWMUeKJ*E!yTyQQ zO&!;tkst>yu!rO&wzo@GqcvvkO!j&DEh)!-!A>;XZ_rcgIzBu zxfr$#Zm&f84JKGw^#Rx8o_VM-r~4F2Ugl{Zk?o;W=};(T_yH$%w>ENOaVX7}&XcE>s}xj?<1- zmjwO*%^jaF788=rR2v2oa@r(qKCgMQK*+O|B2B_4C2RJkr6!E&$%vF<1;6;r_sMut zDg^)_0 ztz5O6Od5P8+UNnn8xh(02vs~r?`um_KK(ag6`m*o=51D#Z;NjoytA5H2H9Rxr5oGL zopgK3FU8@qib)sVM``zQ!Q};~Bit#D(d;651MCtJleJQpLrt}dd4OTr>W0;ZwtbTL z`=fExE@!mVJwiYA113nbLo&$)zfh!DZ(UJIWZ&S^esTl`8BFgdL2a*fNoXH;a?upZ zDNtRz`b2$Y3OQr*SEBG$SErqHKm{UA$d7f+Qe%#^H~@D)ishnDWOX`SR`Pzme67qv zW@w52Y{|K&ykOFz%C`B9R42)}Kn&aLoI>%5!U&_vBl232@PdrhF{inaruBKsb33y8VJO7Ys*s0Lt0Su;?r83EzNWBgPE&5nHzE_?WHEocl|1nR{F_R@tF4$=vzl3E{W%sE)ylk6+by;HqY*pKF7a&yrcKLahV(eU z5Wmg^$;A7Yq!Ply>!^V6 z&=cuefMNE?Vd(PRMvaQ`|d`etOBzE?v_^)lj?Hu9KqfEIQ@xXNrSp9bu06P zU`Tn5TVo92OOJ?v5QZQy`9qLPrK|C0z3Q+G`$I4qSkCN=MnTi+lx0!3iKeR3Mhazk zT2SfKjNEuj9gfts@dPzxL?~#nIW;Rot2te^sB`oa6E{3AC-+Ejl!{UZTYIZ=x?!fZ zU&OJR+8%X2TdkGs7ZYeLDHvZv3yG7w$T;;&26;Foc~3Ou7*=0I3Uq^yp(04??DUxW zU=NXHh3ama_do3PjJM^ok;db(Eefy`d1zlEE$c!&pR!&Hk~(2(koAVezl7beDc~hK z&=&{kaIy{EWZ4>7!fN0aTc<-mQYCNq5Hi^8=#CGv(1jsw07r+X;_!SpYb}am@uGZi zGx_*B&}|g!Ga8ZJ-hH4trZoFB&8}}seP}3UZTKPzV0fk$aL=_|Zy&0R2_s9`;bh*) zUU8iW4_Z0Bm;a&XA?+*Nhc+FJ4kZWX)m@wuJcq6rg9$keo^OU)>q6u#gPM>2RkVCZ zBUF0(N`xI&@{0A*(%xId=fvZY@{O>Ul!=fD?wM4EpfJLhZ62H>2bYV%$9u*;&J!Ii ztfvl}dvw76<9$;?H}~x#)j`TCjXBoiZ;xon5(p^0Nqr=MC%Zy|f>GGVW~oPNJ{zga zbyNz@{JabVtYFUZRkVn`F_N-z%d1Wmk$&3zT;cBmh*iu_ta6AMw2tb0j zX~Y(qU0?E^vska&9#AZC@mS|h-~@@5hN|QFhNwOGb&OhzHFz08(KPQr_e{Fa8dvse zP9$4@YkEK3e=XQAsMZ(44}*x8=Ba;S=zb&RY4()PDs+*)GLrY{j1HsEzyPVDV8Y~l zig~uww0j!m?ERkI7?-jAN@aPr_-ERI@P4O6?BuA)?Aq^G;g7HehZx`bK2PyW=8_5_ z&G?sgkMG8lFfY)zCAJE-4}MTi4&U00xV-5hSxL=Q(Vow>p8jchUy*dyEhjD7cFv%3 zi77zat`r(;K+g`h>B<^3 z>XlrXYu6q3W7}4Y^DFgbv-Gy@GFJ!!qb&39X{}w^^0&zn`xnj{a;eUZznz(n8_<6H z=?%*Hw>SM`hWnA-`fJvTx=_K!Drc#8BCWt%&{?5c>#rdbZHXcWxqRnz^A_7%;44I} zQ+9iyV2Q;Csluz!QiVnZx@8?3pG6*2dXvOms+^xz%{Rjd>l|rK3VTU6Wca!wvp*f| zwqz6s*AR9S$FmAjF?ME881<^*P?9ct8yU}>JeuzHQLt^B-d}B~8MJXZuf~fr?BgoU zw|^$R8M`0VLS9gWYkI2pfJP;=GIq<#7a>+)9-tfas!+&hel@MA2r6Pv{dL|M$EuTR zB|O87AXdFq@k6VZKFV zVj_<=AQT06{Op#*We(%CaURbkZ;0BZSq)S54hZ_g`SOXM) zmgtzC<;Cs4N00PbE*Kq6I$)tEEW(1}j=57ok_t;UpAoFVVF=vAB67DnGIDiAR*7Bz z!b_j}$f7a=j>HXh+(DCMI*RhRV~BElBLVze7+`kjvSJs@H2Q_5A)W*fFS8 z^xl~H=<=;~~ZPbAL_7Z2Q~s;+(md8g-nX18W*^qOGp zhffBhe@uI(W6C`<6ZNY6e zQ6}X?zQg~4yoWQfN{R$AwBC%E%e<;sS%}=oT6WVaoWH&SeqlOA(5$WCHH_zes~r+8 zDV1GB8vthnto`$Q3Y;8j4p`Rhml>}5PZT~;o58{ef?EyF@howU)&4j9vO+<#eFc`k z#3!~gRB}H5*a%%eyn+7mI$KIzX=G~$_1O!qem<_e)wSf6HI`-QlI_kL^eb{2O!vJ{4NmTB5es!!uX)l5yN2cw=ZoZ?u#`$g z?naK056`4*Q)j-RrK~cm{_(hVbINs@fIt~ezO1Q1F~f%2g5wINvlV|6-c=LetQ|~5 z*Uvb`jF|;yD0stHMrbrhB2htsp|GnF!U(1zVXF14(j0_Ae@HlAFYS4wNgipOwOdTp zt}R1>G5>^ma6_;TEZE`R?)jSKFTi6kxWM{@TJE~PSo6}$E(geq^pXv2_r&bm*m=95 zC%O|Q%V*I!Do|r7w}+i^;B4OgLM_~UWlX|}q5y|kp@W8Sy^9V@MMh2I8A*UDOGKo3 z@XnQyRf4#QsEsxtnG0D6g*p00#@ui9VnIBeI4T?(kcVe`rbF9)Cx?;PI(zujFol7& z==~{T*Y1E))K?^$`gGNJyBUtr_8!`VgYmD|XC8~S-)#K8TPFK$$84-sQpO>W>2)B6 zR#dZT%W9J%R~gMTy%R}qF7&(?Dnc6BEVF)J`)>6y8wFM|yqrSwP!}71#;6>=XWTZhA{G60^s4G!G#<#V{?`-ubdKHp{?}F#W!q4M)wovjZS1 z(yQ?VLPGz=ihD;PIiU&TAaco~Dqg(49E6T#8!9<^5#*nAudcllG1wR(SC8-xhc)T% ziNovKZwi~fb10*bHIW`$zo1W?5?;;tI#Q!`7$_Pj<{qui?3*Nvj%SjmhSY>*- z*dDv%z!)}aBx`6Jv=(H%qh7!1LSeV7i%%JaOI-C*6>x$cc--N`@9}JzVLL75H1{o| zkM2EoPDL}8%R4s86y!Pl;XWm~MR&8e=xJpJ{H0%DfkhY<<+HBYSneUQS?Mz#%Gci> zWM9|vXI(uB4ZsG6G5w;)=wUoSQ8c!M{dLsMxi4Age1v!4Ncvj)EWGn^E zUqPM1rz6=@5rBM~RYU~MpZCE+zg3l~El+rzJz!plO9fWTF)SA-mQ-3Qt}Ml&-YJMK zzT3Bph8ujH1)da`wQV0~RkZ&={R8j4&ze-39G6iio3P;W36bD|vey6nXW|%}}(CQ*nNo`^? z4Un)+pU`{SZ~0U(fWbU*F!o2oHH{6%|RS>A&JX-UnLn zWt-*tU1}QrXyo}NYI5>xs;_X+!sjSK4~dA)`PnpC01zog^=q$2*%w+;D*Q0qe4&dx zWmH=QS<2f(8MhfNp^Q2$8(<`r@y)u9De)6nAW-})s{zinGHY*W+{gY?l`i6Un+lG> z>$VE(&3tzp z5G8W-UQMkeLea+}TURrD``HDii^kWa=duEQzw^ss_%65=sNdg@V;S-GO_1O&oCf%W zCqOg3BS1c->QSU2DSRoKJ6DR5#mU7I_NMIv`eD|hAI+2au&%!!_B7}=5QTwyhVzuN z{YU^~=?z*CQL?H}k*|{^%HE?$nf(L2vEP?j8uyH+=T-xXx%OyYd{F+t;@7KFlECQ>r*Ca}KD=qgojgw6;W-KJZ z@^t(Mos>vt{_E$nqJ?f0@a%?#Qw;Ej4v}coP@? z7Bk=bBTobSWJ_`Jlx^*Hcl%E$4AP&{_2*gR3nz<$L>1s?%Dw}Xt9iaHL@p=Cm(5Ab z*FBnJaA;{*%q3-h-{g;e>QxEi%=*Penw%WxozmE09QJPFlCfMl8}cO4iadX=aQdJW z7Buw*c9}8u#xSc;fLg`dy4~QR+dU`51fyTx=rC57Emrawuiuk;NQ~j79d_=U+P8zg zYKEb^%GH}b(cUWJ1+350ns`UvNQ+TZ#O2t%WwCjE>b~4Wml?E~ZB{gL$Z{#qsrtt{ zOlIifcG}G-)rz08Z_OW2c~CN3=XZcJCWH**E*4*0q#JXRF2~Hwk zR0R@+sQlo_TQOGoQmMCSbpZ8{6L@WYSP#4C^kE?ox1M|Os zRK1I?FYO0KWA95RU`5%!7oSgtm^LbRf-jmv`L%U)$!iBMN}P(6MdXwr$(C zZO`1XZQHhO+qP})zUH!g5ucn&s7*e#JgTLls-X zWawdthk`xjGoD{`7lT8^XQhakM@q4^ODF?ftGwJ$b)(6+ zAAfD8wd??fTlsu?yDN!+Qtd#7ZRsi{@g6*tUijixC~YnH}Ba5(&L@D6I#g{*!Z z!oG{=A^q-xb4KaQ-#SJTq{SteEJ&CgBcVbWF$;abR$>B`Dy3D%4peSEPKTqX++V*u z928wBD_c(gB8(F2WseV=1HYn0W*BqrSVCxIM#13?AjrA2@UYFT;7{s7UaMsD^nDJ* z!@4OeZ>OQuziHEvNME+uOYB;c>W#ldx~{&bV|>{Kh`HrgGk92dP=R=311}R%^Yghk z=IoNl*ZqeMrX=;|C}IdWv{bkQbgW#C5-9=rYY$ttY&H_^lj6p~PdRBxfEWxDN)VWk zJ6KiOv02f z)*t}~HEmq)AIdfGTL$Jk=U>1RJQYeF*Ay1(#RTcSH?KlM-Z8d{fv@FG4g0dSl!hhO zxTchDGahl0X#M#)o*}E@%F&1go{mb>m<+5$PIxJyA&i2XeDzgb8&269#%%kR7RZj) zDsQl)SYdntmvZ-mIcJ^?6M->&ExI!xYt$m@la*}hLlj`khxJK_lmqZ!kbeH5>>~JX zmNi=0%Grz<>;MFSz{|X;xx#*-MTn0NAgkfGkyx>Z%o;>5Ys%fx{H1m!ioN4;*dw67 z{Lx+1a|3!b5V+D<-}Nn%=6f+vkCvEH2n>7Iz$YLQnS8D)FbN^+MBfn@4R~I_wU7?`k(g{SX zuSMqrbke_)c9`i>kH|YxG6|w8+2kKZkvyc&hAXFJR>mF6<^tI?fA-BX_X&)Dp1d0Z zHPm0y_P&DE3FiW`fJn48^}`0?t-CrlOO?)cK_8c$jJ^!yCcVzlLUq#b4u5-}KAPHl zAdpyh@9>48OG93m24gZ$TBgSsyJ7HI58G$!QVGZ}#=q4;%>m<+7o6mCtuDP31p+-G z9bLY9ApuIkn0PTHrn6wvU5AM2#h&mL0#p2H*0yD!!;PDl4JATO}3w%YpOouH) z-v-o>C64if_TUoHBcO|I#>7rf(jH2DX{%KVxP|OnnT{FA2?Qeey}xAg=15{ZwK(Vb zJgqL_(DCjpU1yuIqWq2y1Fhkgf*x;s&^`)&CSv;e_*)6fydk@)`|G7;_3sTR?~36s&lh$!_B4TO zoJG+Mp)IBzl~jO7cVM}3-ClFdv?bR~ibv`05p0OT^eJY@p@;zVXBQOoSjfE={Ud%F z?>r&XoR%C_-*=Fki`cR1@9_vxEl^a0?1-o1Zf8*uesyQBFL{~ms;%p|l`ts6VBRIe z0$NZRxJWVIw)lt*EdKQ%z||B^Y&Yaqy=}V0cS?u2tvm?Ng}GFaI9U;f_-G!dpY-73 zpK3}rwP<(XC&Y+Io$Mj7RL2lLZKp@Sied1(I3zow=PIKnwaUEk$*Pbh zMrVZNXdE&rx9eWsdQ4|p>)ZCoe zeoOSTfSMf%9~#egJ~3qN=~K5-Yl%bRfYOmnA#%*VUV(Ocj0u7ydqP@GlZFVd%eF=u zk#&m=s&pGMoG zCQ%#)zW>j*l7AmP#IYAwIJ=`(j&jyTH)g;)ZuW)IIw$`Ulw{CkH_7s(_bvZrllxF*%gb@zgpXbpul>))9c#8s6ord+ zCLwMJhOj}?-S*f6m-U|6yK)+ee3_rX0Gq&cVrCxQoPBb?s%2Bg-MV?)KHs>S-tqxu zQoy)-{5HNiO_eJIrc6VE?t%NTQG^qY z`DYTR5FSu#C^ZZTp-(&r4&oMY^wz(|9axjld1T!3&`mBSWgLLeEwV6x~Aw=&VJ~A zKV5kF=8Qe=Nzg0qCOFhcQmRz+05v_XU>hZJH9o9(PdQY;@5Lu_ry*d^F4%{>*4Cz9 z-PLrgW18eHiu{jV_owv8!+u@S_WTKy0Q>_%20}u&cgR(e;;E~R8FGZ-M?p4af`yG! zlautLJ$hHauq{?iX_8tbP-vA6OUxOa=4Jb{uZ>SoX^;n3p6dwX`+4gijKC1(p3y}T zJl$kvgg4NSY)y23=3+Li?Y-WcY7L>6Shu;Xnz1#DP%{0Fi;zk%Pq0ZH8C1r*M5;6B z7H|@jO`C82J4FNt^aIplB(+90`m}T;+KTiLFks(H;YEgr%qr1iUGn9?92?ArO7TUZ zRV@c7x4o=IXda`C;g42a)Q)+|*3iaiLAj|2b=>-ipfLSVio*|dCznxcITwe+Vv@I< z1h-I%-Dgd4{)*LEsyAKoGJMD9ZO1!o(`YDT9v@G8ZSP#DAbigMggq);{H&W{gQ;$t zZj5Ykr*yfv5L9PLR|#6+K))BgSinP0UOCG@aJ9(VS|fJ;hGP_GhbowHLHkG5()J7T zNiSchsYUVCSN`8OJh<6G>X7_)%uNAE=k+mHWBhjEsN7zKV(VUXO(M%o%gGTba~mw4 zp&zN1l$UU--AXhh5^QqYTvhBDRfL~!GIJ;a z*l=mWk7LKMkN%OThV6<6^g6z1Ui<5(^GbP$Sm|2sHvmVH9feK!rg;T*d`0tlmzO(b zS3XF6!JhY zebm+GKHdr2w}r)mwck4o;Miq#6d#_80>B0TE9c{@yiv2Z_}N;_c3cQnr_j0{9qkwq zSgPB6@A1)9fOx#$X}@Z+=6Mv}&wWtI`7x6rvgM3Kl93!{PPh6y4+={Vl-f5`M=>Gr zAFVDt5ab*f=oganK_9tqE7VKZFn);qF;a6oq7NYmrSXm^1l7%>`X_I(avtA(JXNRa z0re}JWZ(aQb?E9D$KI|Y5mhK$=ISjK!-dC>(A3ju)RVomEK4Q`CRr6bU%{I9oM?j0 zph(wye+(EBtV*u78uBjgjgX{PNdAZ@AnQIbdU3Oan@YUl4qPEzp6Hh~TX0-xG?%bQY+F?eU)H?a(1!6L8NQ52QA${QYr*;)CuxzghvxOqnmO3Qj% z+X8xnO9Q3&X#fCm8z9!F8QQVw-oL9Zr&=*ewcBD z%}N*C*tPzMW;Q^=sTF!aSN?6i?skp=M@VTADXe$ki_5$;Ve((c@I+s7fC{8!EG1_95JB)(X0Fs4f~hsq%@RuS zm9kcAN!uT67RryjFmOxNfri0rM~?$80o}H`pAUTYPe7$m>9cEo>i=^xHiB@91~0uw z=X+p<@3P3XXV+3swQ2k{jUZ@j|3$bz7^;$aA}#2r)~F=?JisBs94;)!wi?hdzlHKx}UL? zoapw~vFAEJ0sYJLD0#lwbhGM4Y@_FXVnUDcyX~x|b2=GcKmVQ}@*AxM$t@^uG~`e4 z4g$;w84lop76gwEX=Rr>4f@x-6Bco>q${;=1&RoMDVx?ylye?DVckL8a2*KI<2zdg zKS;?&Yt?|Jkr7Epp*3s#U5`*^`pCgsAM?y@1i3yRzpzArkdYNr2{wdR3ePK7kVr36(SAOpx(w0BzU6r7YEue&T8#eit77| zkxGzaPK4n)AXmg@*K{Jw((U|~@;)%;T8jKs^&Uh-6_VN_boC;aGt1EXR% zASXliSgB13-o2Z6eyv-3-o*_MCmq4Q>Pm?*ah^PzI7|lmZQHF{y=-35FfZr?3Id=Hjroyi1uzD zUUp#yRZhyQCdGDPK#tjolu5Z)E;8B(p=%|giy!93T7ybVjGBr|(v-UA3P&SyGLs1-*sMlNj`ooS0A)*2O!SHXfO<@O zq{@GT>+Q{*8d+D@)RZh=)8MKT=Y(3sFf`}WW@p0X_m>iGmLa6SMI^VMQaoC%GuhHN zNmw<%h$kM=9mLPZEDiA|&NNi2x2(j~*!i(4!K)N(-%xEat5%G=Ed1@)Ry~dHUpw1DB*vB~G0q+|W`4l` z0xzDgG*joSBm=4EjTUUagqD0*$qHbf-y3DJ`P{lIqDIVtkcIb^jz~X}G)4CeNZG59|$WJO^gpGqGfFdDX~Q`=3FJ6^aU7IA^Vvm?k>nk)c9+0;0n* zrppvmQesn>Z^luf6Fzbtb z+rn}*Fj78(YdrBp>Ub6!V*!Mp;gDK|(;(xT>87L2>X>FfD|#?ipy*zCx(39~wv=x!|OQ0IIAe6)_XFG>pw-wS6sW&H&DZ}BlXbZ^}CIcMvH8mpd| zU3_2yMl?3GCfWR<_m3r`5~6DjaYZjn-oNyY$$3>VFF}>es@blJVaPhbh+Mi1WccN_ zjmD<#-&Xw2>8@_TICO^X+DWedtuG7aFOr76z;0!oMyuz!F4`z=wF}K#Oku{^s|zz_ap=-< zOk8beSNn~dS^pd=J0`UN-Q2vzMX{y%Of-uu^OR~K^;$=nO7{)Q)V3KRy(%<-L?som3z?BW^LKr z-ExyX7`G_&5WSK(V!mUV*5AXTree70RiOPiJ8?cmKu8p&zXt93CZMYt;K=qoyGiXbd~9s znxQMSg8E=5Y%L__9+h$W)Ke@Wexv##HxUgrYX-MHW%Po}qASTG^A38VaXmi|w80g9 zcr)iBZAQIf*v$~Tl4n88C12DXN(M6Hs_dZQ+bbVmW74d8M?bDTz?~xPZhl4c z>O!5nyP|#nEw%Qqikgs)>DdP-)FH+nGr*y{4{K@lJMs1PV#<4$UQ{EOKLeZ_(PaWC zQodj34HAW%Sg0js=x#{>i3|-X2=a15#zvd6D&0f}a3kW*we&ZXJd6+0h=bS)4z+sM zxLz0PZIeLm!N?$;7Lo+7@7FxR(Wb3qTkee1eR=VNxGD9bx|I{z%!;GI%eP-ogc^*q zOLTn-djD}LC=yQmQ8P@>Mat-4@-eSMA^+w@T1lh~o@8=G4)@dGCxuG%bvs$^R4D^D zS61G5$AlEt8qH(z63Gbq+^w zd|g4i34%h9I&l;h)qz+E3$b9bZj2!+63yc0>4EEPK`~itz=BpC%{Fh$I$G$UxF*u9 zuF0oms*cTu!yBTfWrN^f^@Rc%69M4PyIjB$D+dj;yBP%6)_5o6vsSQ(GX&Zszs zVpV{j2O7UU>#HQo-5<2}l|BpWt`mcz^v(%3ZK7`)&!H1)Gy(ckzNpHR`aSm-@2Lyt z)P22OP=V)9QK|aLWDk7EWZebu2eCND$5)+{ zYY!Wh3%*eN_C0fnmjbVr;rRWlW$Eh$OJH?}Tk!lywY{64#|cyb{Q z(1=~wF1X{j4sGy2hZ8=ID@(2IecmT?1F8+0;VQ-VxS46X70xjSAM3+}gbVExif zUEbpk4WCRpuJDk3TV@h*nOuIa82kP-P7Yu}rHOGacLLeRi6h}T2jc}~K>xjk9%@75 znCZUy`sbzjJQY~n?K4|6qg}pgFoU`u4?q!PZ_RcVq1c#FVKfjm2vL^&yy`wB0H4lH zNmaRezrGUih@25o#>j?~1#q5HF2Aq$p}E{Uq6D6$*rel*s8a~K)~hT*L6{*0IKsDg z+5t{MDbwSH8=lCrU-JY^t*h(@U_znE-P5Kjf%wQo5>^Te6*H{IrhDI?^W8ffu5}|{+eIO+L|+_x1oxt5){@6X z`G0n*R!pasFZq;nF%q=p%U1jao% zlPYnT+Sj*`yC=S#1moliUCtel$8rApmvB6-&%%bxl^Z3>(nv?GY>1Hw|LgmV~5DnNIX+=TTt9b-&F!S~@IMN$5ZFkr`m+!!EO0%rW8VJaaY53JSq9x&t>M^RNUsiUudZKKKeVKBl)V z(3bLMhB5SMFkud5tJ7E|woj`{8-so`d zELf>8LCAA;J#lF07vqehU$78oQJIYt;nN}s%I=K8Gu{?gc&WZOb4Aul`%7Y!Rh zV-6?GY3eZEv0!1(38co#Ys|@I;;ydr@T6VB=Xl?!>?&Q2S7j!Yxp-jyk&5jB zwq6mZRj^gz<)B|nx$3u8{@U1mc<*gA1C*v3uZ8?<3k6m}kpGrUxE%ZCjtqY<9+2jU+%h`D|=1$Ir%hRmKorD&Pw94Cr`dSd)AS zME5~$U9zhmb5OpSNY0x^w=QF}X%k8+9q<%gtSdb?>*>gK({4Ws37gdK33A3dkAcu7 zBEnruGJoQ(t6aVaHZQEtQ?5M)JTA1*Gy?pFD}S^SbUMtITVu5*cl1OYWgWQ7nE^rv z&fPpv%45lSENrOiNCSsLyr^iH;d?f?LI`gnU>n$q+r@N8M%UWjp;0oHDdwwSnHXa3z4C#ySCJPCU2lb^NWMh zXa=ppNG=Tq4r6MM=qMB@KE~pITkADUn<<22?3k91#;d?53`W zD@(aVYmJ2H20tlij;@I2mi4bdl}UFQIyAq!@*+^#P2)8pA1a+)r=izx1*jc(M2oN3 z_(KB#$c@+)d3k-0Ezv-#ougjlnG4Jb982*ZBlqlVL`>Iv@7-{eHCcHurL#om<9s7^ zeNqGq!=2TCzXne9Mmf?Ozz z6wU|5_{#m_fC$#1eJZs!C2V8%kc1^CI&bBaR>zB*8;116()qP^@zWe+1NB(T&QjZ? z33F2-?npo&UIabxo1TgeZJm}$3&6V==m7b(BF3m|ySlgD;r@YTSnE1Kg^}i$J+C2f z!3}7G|F>_$kPE-U1kz1TT-$;hr1?|*lue|=5XpA7OFBh0aNDWQodk0{r3D-mkmj-G z3mWmilQX7kXliY`7b-I{allvCjJ!P3vf))ejN*usK5fI5&HPxir6~klFH>myD0Oj_ zFB1?_&j#lRZy(i^v13#@K8b|PHX&iF57$48`6Z+l%7Nk@wO%`k1c_4^HaBrF{c zSe0+8wsy;LwyL64uKzGdpFAVwTSB^PugHe@+WORWYU@*=a;ZHrh0sM{ISYa3FxvV} zooY5_0pA?cJda(?o6Rc_-8ZZ!(5C@NYLSP@ev{^HsdA!?x*vpTD(cN*e(?*fZ3Z^7 z=L`fl@IAqcN=Hs!5oE>Z`((kyJ`m>~-%@k9LK+=S9&rzVH3+E$Z)^f6Zk(Ak{ED(> zK&07mYb;XxS1#e_a4Q>+wwWRESfc1X37+8JF6+onnC5jvZhI!iah0DAQ;aLgbwb>& zl9QTC^s_fjO}Pa3fbBQ8FZ?Z9xZyL&_m}5~v?<(3nUp4L?#*-T?eHO&tGVW;O_*y- zF*%hRhvZwdC>{6@9H{TCl$hxFuDv1)&Qg==VPwa$5j=q%O_XbJbtJ#1!8v=R8#XXGyn0p5WxU}t^b2A7fF_}0;d==f#@c$(uoxLoy^BIbEx3HUy^rMwfYw6ZrkuqKRLC}Hw<-nJUk-p%mO(ODod40sHKf7 ziD*Cc2yRjYnDZk2pv(nbnM%IE$Y2lCgAz*ro*?cY?39u@E%p?C`t=$+l@+nY73nRl zXJj;e+0>`W8Tbr*wdd7u`fGahjnh{%5dbMKA?F@xI4+ zLBwx271w0fS*X@!EU8r;{vcJ*X0S-`g>yPzj5Z8~mlZb0Mtel#1-oUk6)HflCg1bj zOp1DN;?%Q8(I3=Ulj@R}i!Cesu{R|R%wxlk$cr{O@2bf_NOr3Qt5kRn-|6!blP$L< z>jX@5(&dKQ%|HvXUa!K*HuR(_Fgg})XFJla5$(`0pV$$ilUDJ}cd1LI5fPl9B_^FpzxzIv6%a{t)_ZPi!l@7}AR4vDnhyqVnqu?*2euBqc%wKBcC_3Ub z>n{#vpf&8}M^V`60lZZ>O>HmrAu_u#K6Y6WUrO>zGjw$?ZR#8W&tkabWDj0O_xYR{ z9*?n&!6Me<$w#~PWixxWT=p9J)Dg~TK1Q5-F}UrqLH!LsSuI&%M2$1WDH(gy9hi@< z?7Mze2ep$lV>f`sPZDOqxI{-cSDo&vxgHbofTR_#seLY5KOzhSI}?J{L3i!Jx!p?& zkb3(rKPe7Iw1?l#tj9~2CDr98LPY-6e%iu}0Yxk`X8z0+yvi0_Ggd@g`I9uB=!?9_s9ccjCII1pg@ ziT7I)`{hO`Z+LM?^{<*2QHP?gwOhlurXSs=S@#3U_G?}67+(?i#)2kw2o1p(lR+Jj zv=ON@;Zc$vZmoo$o_vuvua6$|I`)g?QF&<{mj#4izwD-*+mPL?w%%=_xhA_%p7qZ_ zHa`P9L?I!ABqZtfl=xbDRxkN=M}7O^GFkx4&`m82A_U?QO$WmaMc|_4GgBC?CEQ%V zcXC!dhOAbdoQl1rT!K{owS~3=`lfiTmk{>Y(><(++VD5h<4aiZ`+qXJhGVT5zKb1J z=W)U?!e>6V8HPhhTD)r4r=IeqKEWG?_(c#=$fYARN|Z4Nsda!wML@2WIaTJ5J;Qs1 zQ21>?@}}`ua(!K)B*hsv6g)5(gdHrF_L^Vbat_Pj*b_WinWc~67TKA;(vzmj*tD&_ht3y-Klg}nSMA%K5}+hgRUHiD13 z8#d-BIv%HLWKmpkgchQ`rE~$gIh8A3NyBNl;0=ykaIWjc6LC3#A?f|sTsAy0h)xsd z2U$snWWnDOFrM8e!!~A<$sfH))u0Wj(YX1l?ZPk}<{mq1`MqXnk0X{ZrMvU0VT+zt zKXEBucX*>7W?AWHg`_RK0gj}CXW?2`_!I?T>{P>WaEpTp@Bn0R0~~4jz3g$)TqneO zs8C9MAZ9HS^P&+wA{3uux{jI<))_#4RTMq83qB|`fEK1{3j<%pDszZild@q~aYv|? zyg*9Uivhz0lnx8)nkq1ONWguUL|e>_E`|^`IjKUcr~{V5C~Pr^qGQr8vkGhEKLI2T z_^BZtfuG6~kC^KL`a8aqzBHi{aazLlTebuH`yo&XQmc6v5@}aQgQaWJ6Y?}7U)nBl z6Db2@<(G~B2zGAU#5*Gxl$zF&@<|T+Z1Z$WYwKXHT zA{Bk;6oX7mD?^`b-de+FwD7^JiT`}(CCSM$;UcGmu#vgmv`UY=?h>*%_^K>lLw$uA z*y2g4tPrSe(XqE=xFwPMEQ2<6cU#d!-I9b24N(-N5eDGTgYYV;Z}q0zRNJ zKJI~`ytDVP{65O$u2k18+XI-Pv5j3c`)~_$p16_g0f9b>)f}nd9?T|UrHM0Nojcl# zdnZRmldl+*{!E9|C|n*C1Em>Qer@Fa$0}4}MUPVDBE{ZQ+4O`5zMng6pFM~no!K?1 z5K+UXbW&cD^fMXmASh8#Gg@P=ofjGHB0Xji^9>^M$}t*uFAd0l=zZhs7R+a5WDe2~ zmh5C@idD~=ksB_a%3H)kI2HKCU%xNHP6-c#&`I@15hVY771#qmp(on|A$>~|qwv^yx zr7Mb`dTTAYg5k*DUkfdK*!hXXwF-Y@q#Nq`i`I~U!~!+Ea|QdTY0*>LhE)2wQ7-Ss z4ZP`IbTxal%C)28wjfMG2i(jt5-(b%(tVns7S5J9jI||P}~bFe2D0dBRGYcvy&g}`>N#iL8vr?QF(95 zq#kpi+%dxvH&QWe<9vM023zt+s~7!u@~P&UZC`B6xg>u)l;FFGiPLyIeb#MPZ(s!1 zbm6|Z3da!kxlC^tn#sRB(nM%?mE)?(Uea;I4rXyS@bOMx_eB1I^wR;QQ)@r z5_G7tk4!b}2S(nN22s(+XfNEwEjiu(42i->skI)u8Q2_@Y8LhhhpszgZ ze+9{*0-)P0e<@Bgoz^e-+8SliX7ZRnX;&e9p#h~}UL*!$!u9+dlG&tDJH_RS3r9N0=Pn8y`MSWOEAL$)nNK8I zEM_dz#$dx$tRAOb6E$5hF$BbE`DzqDhE@4%%4l`s+Y7mReGC zA(FK@A1r#OIvjhI!>Y@AaKrj6tvH#2f}Ta^T^8#1{arFh^!Qj1f-dSpFs>05^S{!? zlU8FN^RMY5OW^5Tu(#0#fZ#8D&nQ#BRSI}-Zn@aw2J4BsZS#}GxJACn7aR4&!@m6wh@gy`6h{X$b^ z$nZF=;&gnI+b6(XR5}F^f>x{xRh>q@iDJ~HDw#ZRyt=aIS|K0w7AYfNhkllw6 z0bQe=pAlVTwU?fFqT%V`0Y$K(DZp$2b!E8A<-J#^A_A4eY76rXIpmXeyIh3zuroLP zB9-T>Se-v=+v<;SUIP%O1bPi z{d@hhUDlvXNN0Nk-9cBwJ{fcqYfcK@iQD$jV2PMomTHFMV2tc_eP9!<)RdomA$!4A z^uj}m&QQCa(dNUGPN_$3oAgN1LhxZ4l_J1G12!pH37>rot^VoIiSVvedazt#js>6S#<-Ynpd?6upVX`7!G^_jpt++# z7cq@(f04e6&7$?Wh@Q9B;I4ll*JJFicsn49{HKHLXLbS5nh`BeGy2N442)2`1ovkS zh_{?41Gs3<$cZDzOIp@Z>5k|=Ydj2I!B=d9NRG<8A1eHy&S%f8;augK1U&pSZH+ud zWc8ckEl|Jo>^_r{aRGoz?kusIze2D&7JL*oL zlT0WJVow3O!tSAgbCV$KH;u(uHQ5o;;CJ_Omtb#bm;ujQRt-Ux0KW^0UjJ}_Or9*W zQdE6XWj;9j&u+K1mERmnFxoda&hdT-AW+tX)(fb&#hSFmCSEqPhV{%8<{-8Uv#6CxU(^4LL_Q8D}v(u^TY=Oo0wuuapuY`qkCqj|* zA#szlPYLfvJUHhMFNRhHiF3K@Wml8M5yQ|UVL5)-JqAK7WWbaw2vPPLDv}Re_{wDR zDDr9Iar$N5MuEYmBZ<$R4Z|qTJc(~BHWMd@LWHC9uCRk+h|4MKcF=B%ruo92wukq! zdVLr?J0X$5jtYa0-{PA4x0`rni7ys$W`B9*;zSSw!Fj-h42SOIOwX6m=-czoK-EdL zg!#))Q!yiwcFJ$SAk}Cz|JM$|jPKoj|8)o!B7ATA%kVBxAfyxd;6E1MdFc=SqYdyo zFZaPS|A$xn;935|D}C^+|KU|Wc((uW`X4;|e|X~$p5rfXZffTAK4+)@&)me^dD@PA=geei!_{L%Rf!}de|7ly6P z$F3ZILD+rpe-YT}I~dzo8Jm279&*kT~eyn?eif z+ld+fiTS+`w2<+CR%jR*7~hk`O5fCxkm-Z7wQ&*7zDt&QI~1q~TR5h^Kj17in$Cv#gHMSUAb8Ub4?BSJcHCw(h(Lw*}mE8}-K zbkh26s^&&c@8>}OehzesPR7BEbs(fu`!CM_S70B<@DZ3M zAsY+*|F47b{Z3$FVj^UIzk^syacRzph-{l{=|MdQA z_o4SL=U`_3L;uM?F#N;+2g7^3AF>bZAAbKQ{g8jiKF0ry^@IO7pMPur=%4fX>+eJM z;p+p}f6wJTSw3_a->b{Nv;4UBfArqzf6ww=?@wm^GyDMZNAMBFeaXiRmjAW-@cA+MIG8`1ziWQD{jmO!|J(B){Q&#VhV8w}IuqM}<{yXmq5mOc zVPPcvs0b{~9DigV@;|=bOYcA1e=;UkmOnD~_xtbP_!GtdN`QaK@CW^$tNsr1&sF?O z|AqUH{xkTK3Gc!DPx=A&Uxxfw?{9|u%Zz^k|8M$nrT@3=WBza7fAn7#{NL!u!Tew4 z?|weyAO5ueN74Hzi~owAfVq>SoUwzTt+n0zW8uA`v%eQVTPs@!#XqI@z4p178ybr` z=(~T^K|@<3V>x{XYezzE?!S8P#r+TGY;EI6$jba~F6UruWN!FToEYB!{=oBIvdpcV zjNePud*yU8{*Mzvy1!+MnUUqa#5o%{{V8tZ*7~OJHSRx^PEFRp!r0J>knTU#@Q-gP zV;j?t%E->~o+$4H&(X=jSl=4PE#tIJGL%F;?fj96DOFRVq{>c~gj=~E{^txjD;WEc z=mH-NuEuCkCu_5L$Z)7WM(RyP4kmhp&S(9m^qW3v0}Svs%RG!WBxWJ)lqD=^&=#bK z;2O0^)bDk>cpdx*b3F2D?go=dKiQ`F+EY-V)ULv*2@On~?{OP6WPFP-174Fnwviz-N&i$8 znlQq}5tU+XDi>;kpTWfupHdF+F>7GBh;*C9t#t@|O|Se_lF=96o6_1>qw~`6P5H(8 zM{f!`(}pPNJv|W_LHp;m!NFcDx&X)CNHDRGYDF%SdN_?liY*+g) z49g=q!rZpNQ7xXB`zW=2a8Q$%mNfnD4e=nFd zXx#d=_DqK@3Uuc#GDpOs(KIe$5MDma;$V(HMl8|_ z(f=w9>7Yz;qSh;0gp6*`BETxjFcr>_QBr+ontjIazi#U4R&jB#TaN<8~Ujg z$xy{V^prBwXBZuH<0O+{45g6n{HV^L0Uo(nl!QS6qSIZSoH5LU2LU2N9UA)ci#3-- zF3MC3_{L99au6^DQ`J7=3NzeVDU{B8S)Ql_wZ7i2y}rIIpwPHG5S(G=iLY$$8F?iJ zr0a8?JV`?n0lAPhw*U;4^$M#7=)%11A&~+4vB2CdaQZ6l3C%6Tue2P%cDdqDc3x24zV#M+HoJ(2|NQhFMTxHBlt4}+bYXO0o zZH7P0K^y@d4kH*SOWdt;-V<$-r8o=hd&v%E+jE=1d}TG5Ybc}~igra87MNY!>Jy&^ zRwM-q)oo6HR{(V~N5M!}_-E})w65OqAzE-M?r`BisHycCXuUC+Fhc{uera=U!I0q} zSDJQip5!9GpW)zPPw!4P^V?jKw@5NZInx**BQ>Zvq!yhyZuNJRvYVW#n@JME2al@J zPavO#3{Bw|xQ`MpoYO9c(rxc@hQHAWc4Q$#beRG-BV#PMLP*&I!xtu5L3Qe4 z;=7)-m!9s7n#DpzmU*(;zpbwnSsM4dP8?Q;PrMeP!s3yV;E^a|nWn}`8egWoI=qH}T2!RAYPr?$MAyc3@siduBQ+B!7uaD8MOD_oc*X;4F-#ri6mIIQuC}giNu_G}tzjvH{Yaezjto*Dv?LNTI(?cEo>UHdW?Y%z*{U=z zy&B1|<;Y?2iTZcZ4eAmwF#IFmY9)(;L<^ZRod^r+`etk#0ZFgH>xI~!5}hGlkzBhQ z?AWE>O4D~#2d>ngBR{8Qh-O=hP;1+6QksA7Qmjs@lbSK3mhhqAYYmzcLB$PbwI#1@ z6POGv*7j(JiHOX~NTHLJ;CefC43efZMhTHo&hEnMfG*A~3h>H~1W%#oZ67SQwH^0W z6V{RdpPcw2UC86SfCv~no%koJ8X>zk_ETln5yKAUXBU=9i`he8DJZjiT zaMu=_`95*Zot8C>;j2L1sMA-XX!k57RYg@2Zg!}{A8d7Y4L3>(zP>S^Ylxj8p20%> zdMQqmR;1!ETs0Kh$i6<(zYT;gI(g7+4JJR0%4>iwxGUxI9PwW&zL*uurG$Z$`vR`; zS(Ybr!3cq-C9r*me-ga-!d&l^&BU?27bEl*IF9bS6QMcVz9KK&eaU~bm7 zuBXGneJJVL$mHYWLn$m{1>xs=;f{ugIIIuFFwM_L1Qr6{iH)fXR%KW>!e-_S6>Y!c z?o*~UVHnM~3-LO#dR&&DjN8AFmnE}-nM*PWqUWS9ipU3dexW68-bOuX?M>Zt%dGxn z8|#HJIAWktw>l5;WWumt?4tqblA|ZKFe`+0Q-DZU7-D!{hO~&xDz5?UU6;{HT~111I*B_AU;&i$Kg2TP8zi^1Npu7H8= zu_z{O0b!|}lU47)gF~m@isl95(zt6BKhlv*;CNH)IEmZg$vV$k(+?)zZed_q5ig!8Dk}XD=@~t(LhDAZe2Jui+LXyUO%ahGJ-+Sj84d427Z{2Awy5;%hx$HlEX#8p54c z{64773`~$I1hAi{c=k_BoB)a5u1*GtKxz21bF-hHD8ofo24~xV?pP#x;N9&o2%rh` zfgMgV*yGPOFm|t=bY7cy)>#CCo$3>XAFjlozsO&6;j|V%m0+6||Hca0x^3p(RHk$x zv!MuBs;cVx!aW{YdWOWPAX~oF5iJP|ula~~S~wtlr*(b4p7gn?T6;wRf+i1M4|A-3 z5{DWn|Nb$7p#&^fwfg%qfw32)d=3zr>{hhTazB>ygE9w)3VS|?5jcH+dzkeh^VF%{ zj|Q$8NxrP6+=)8|51@w6@9$}?s6wP|6+dq8cG&eipCx9^lJ#pYexLJUonRsOiJ~_U zTcg>_UaRU5lO3(jK_SxO;*yoU9Zi>XP@Rl>`gE9Ua^M#7rGps=XJmod27a5u=9}XR ziGX9T{8qk(3|-7Jw*57B4-QoTH@n1ambTKf*QaTE`1wv2>j-fRzkVzB{2;RcYcIKe z8pl3lSW`c=c7`cmb-qQp_^?Z=%TOf^@%QI(xZM2CzAxJtuul9>}rf^vBKYr zDFaLS2&k@!;d_kSWw5SVH#4D7OZ@nk=19znCAw9*qXpvJ7C8(K_w;*N3@W$?uYFBj z&8ztKMa2b7NA}-#R0x9Y*-Kr<2*UzDhr$fCXL;S^Ln6k-JvXHzp{jDU&qrpHv{blo zkRyoVej=nI$5&RslF3~bD~veYvy60Rz9XvBY)vjH77HZ2dN_D|EHPZ{b1QYL{6r;$ za|LNQl@T;Ktjq8{zCNQYV7b+#8W`2Z_LdlA!c1ES7b?>gsc&!1Q<<&_Tx)A&-*dz* z!c1}E)$2x3!P3Z>zM$@+>9y9i2N5m$ZmN+3nTvu{U|htN5P)b@BZ_WU9UNz9a0bhE z+S=T_9`j=hAzyOFuTzFN_Lg4eH+F`)hp~^=))0`Ya~I3efk)Qn z7CFOxV7f}p*nG$9OG^H883S-cXm-v%&@gi)7oQ1}gxyoxr>0W>j6mOt8PIjpxBI&zTBv&b@0W$(jSiTFJs^9*aqL`v+|K@3nBH z42rg1wpi^ScKW#s@ggZJg=&Y^1-6+*0sUoh@U-oBZgIx77mI@YI}$-dI@(D{Wu!P< zzY5}^q#Yi19NKdQc`&xxwpfXhq5>GW#tdGzU$Mx@-?R!e=4aaq_C#6sxxYPJ3Q>>( zbNS5K%!;#WCLfl@eHU8duP0(V;@-?@r<#4cEmF2XZPOS5*j*cdKg}NaTKNt;?rw_Z zH-TQ5_dHQLwC(|Cpbef%?@d7PeHZ&Iwlx5YMoEMu$2DGHOhyDHEp#a-o+I^Y!GP0A zgE8rUZzyHpTRBuP0N+I1{Do9u6J?$fS89GiR6%IRS4nyeIB;E6341UQYi{#Z%Sr%Q zM#N<;dHQVw;Gw(E}(@jcyssPsbN@AP;ZNrZZbTf)G9BpYM9C{dNyGL6NESJoeta49{^AblP-koB zx9;Jxf9@3LFiy<}qHS>q?oiov(wnKys?VtZqc6(PA9%W{>%^P!m8c|jLPLS`78vyz z^ytdiK?`V+Mr#fsafbqmzfOM@0~lI-M*f;l?dKp&h!G~eA@4T8z81;4bL)9>!E|*8 z2hOuNtaTqkv?L;;E#{nEiWdmVET%Yz312-qh!ClD_ftBT@8^a-!2~8wMfOT!>44s{rRF~+<5ujJhL57`_;<~)5Lxt1kPeTf}M2?hb^{JpeO9R79=UX zGwJ{i#vBhulx#c^{V?(((~jS|uK|nR)3Mdjio_JgG@fpRgAT*akQkKx(quyhbK!%* zCfC|F-8)6GJAy@70h@_daLq)}H1cK>D0C^a0uY7^I~6l9-t*^IA|2jJ zj(Dg^i10OnfnB&2iHSpMe1wVqM!9b0Qi)8&4(_oNnG_Enl2+2SrHdLIcf+RdUj<24 zOo2z5nD4Qfs(s{aBgIQQ63c#@uUoTO2zn@OkMuy&AdyyRc&)Upz%E4rh}ShbY@iTd z_ZsCs>j&det{N8mzJV|&dtt1Nxcp|I(l;RT;9kzOwo6LzqtvKtDEHmx#G1K_S1mv9 z3=u50cOjK+XATFXKEZ@}KjigR!-(3tAXuZ?e~w|6O7QrrUof)FpbWL{1Sf? zPUdx0nN^%EP*3Vai)=Kt#SdW4oFc4#nt+FXtS^RL;BeouNF| z49A_l7?b^9aK{?7#JO$S+8+xm-^{tdEizi<$bAT<+W1fik(T15P~E;p6PP7qHg~w7QsY4D@=;Q&8~EW&8#4SM<`KAEB#`@ zP^SOIyfB?@r+6ihyKm*Bm&oc)(U;&$s*?fW5a>i=FmoLFRY(EkXT}@nnvS|506FhY z--ne*f&fr06l)@l_r-$dt0nmh#jnsDUtf($?`XU&yrg<#2co4N4nsgx`yL!kmvr&Q zQ%IP)D@ML?3Y45XNMm=dQ3zOIYgMBCI z)9PlOEtdRlE*O`rxIh!VWh09C1~~5v&?0d(-aLXPEC3@<@@)HNx>x8aUgTvCkJWP{ z2_7R_UH+Jf9Cis1fc3n0i4E*+vSx5;*5>VQg-*7H`(RM8 zfqu|a)DbQ1AQ#-Atj!VJA#vwPiAGuQ4dFSt8tFXautPuO8F5j43k7G08ZZi+Kb0~c z55)&dDlY^=)X}zpW{?vNuOh#mqA%OHkVndLABG4Q zEbRpM7km!>3;R~kH6BwBrj?<|H|8GAL4$2LlI(JQ zl5-&2o8mFkpE}KoCN<1sk2kr~ViUdWONQq+NsRNL{#8S6Z{j+>;?Hd>4{%L!8eF+_ zMTNfzv@k>LN_4^sGSLx;1?vPW{NbpHL-Tm*k;7CGDPsY(+0#*puZa{ zOonq*#5xX)n3#Q3ZEathJe`3c^63S9SShWq*M}ew`yu+2B}q!j>0ohUW&AXySRz^` zcQ&cgp?gn7wYUJ>S0!_0OVmfZ3VXZA6YAl7{sOsB#_$VYLSO+D7DVu>RWIsZE@L8u z7HE02`Z%+>wGk8xIc0DS$9Zz(oa@-2;Dxo=*ew3JzDTNtRfbxaYWF8a2+Q zUEQ1p;L_5R&c?R#g(lGD`CxgzIk|_UFC$wzTqYJKb)jpJ+NosBl0xziTX$wv{1Q9G zrxcjQJ;zM`MO7rzZb4ivqedfF?-8=v@Ng-T;R=B~y4zfFn*vdz-o;JTkk`S4O5-0L z$miCiBr?@TRqNo6ZOlUCkb#vGtVzi~_kdiht-d)$uR%0sO`~5?8ns82#jU_I;e4v) zz8>NSIn*06yn2;F$PYtK%kf81U2P{Z!Ppw4vJOtXzI{5Au5YyT7^hRlkaO_FHHcdW zxwSga{6DmvQ;=p&@aEgb+qP}nnzrp}_q1)>wlQtnwr$(C|MNxc)y7`zUhMVBbE2vu zDo^H9mA|ytk6L1SFgI&UKj*cj-OxxJI@Lxer@uGpV)7J-(rze33IF1--KNY#iyTVd zAvaYs&HdrIY1A~dJoOH=VX$EOl`iJ;gj0Mv=;z_VzJ z;v744Ug4TPB&zGyKDok!tVfe z_|$U@wf7EqR>#TF+NNL@Jg!fd)z=yw8{)5QW~!LN=)wuGRmnU3UhU$Ayg@tUxfSJP}ger$6;jjRH{l^J!a$pg!ic(Nc0Qf6V1|Q=)}$n^3a>vB8{wnfEb^ ze4MGg8FEV{%Ywd7oh~0%RH!*Y0!+yJIuZGD^v6X5cY}R)V#avhH$f{BRt{Ix_ym*l zY=$NJ;WK5VGji7Vqv1`DsMP1jW1c&BFD4Tl)rr9saQu(^>>K(}GvFI?qRzVS7k0!{ z1E~73oWA|ad#^bfOU7(>0D|eoU`AYbto|Hvfc}I9x`qzYMKC0zk9HUNoUGs)5huEn znULqiYvA#vqcq@dA0{F;bi^v-;ORvwbIl-dcYNfeHw&M9gV&1TI9Z)UC>OY5a(OKA zYlrl7bY~eMg7Lq_q{`kc3|F?h^Haj@>mogF4r4*MUWvq&T(3x5@a;nXokCcWV3h#y zXTJj1`UUKdC(%m+8c2D=r6JC8@q5~Kn!O=9-D|z;JbC`zitipb4-Px4XMwX-V#!CCuyLr(OA0h> zFa4YzA`b}zK^x&A12FQc?Y^Ddax2(TM7scWDS=47D}n`wcS9?EbS8`!MMnjA$un$i zBPsPm7pI~RbPqPqR24%8G2q8>igPX20~eKd?VX*&?F~*i>olKE>}L!L^Mfm|CeXR; z65teL2@7s@JB}@XpIXRH;WQIlq50X&UCmsbaR!RTYG(WT@pBg*5w{yF`t%Qk(fLg1 zgiJD|qEBlxdqF2an|JJRQt$|#SHBWKwhIGP z(~~#tMt`W1F;7h1s|x~6FXV$jy_Ph**dfHkQ4bZH32)Q3ZZY*j9d71ge{E_^=61!2 zkUVZuI7g_Vq2x~}xmZSlc7#NwplYpHg9|!&GkZNqggTx4*jVW3OPutope4b_x&a%({9Q?xoQS>gr@IVE_OUyhid+VU<05X>wUYwLs`k zWa+)4;JUWsak*XaXN2r&29%cI=;?}t{UU%&o%6l%HRBM><}@O7ERH2Bc}7W8t!(eJ z`~b4P?>nmvufkefMxAdq*p3D@8|-K>s7rGTGHnPFo7s8*I*I6p1B{K0Z9${IgpX{H z+U#{^A6+MF-Gv~Q4}Br6T{+YqrLMsY;R|>5Vj}?kacg5WWsiVuNYX6a_w`#)4T72- zY+h(DH!1Q@1oln@XeXaac`gI{fK-q`Xaxy%P}is7v2lE0%F}*2cY1B$u(Ho9R_2s?il?qc;4h86Vi__j#tlplLumr!xkfLTbioyXJ+i1PMv0-@2}pi zs>74|zl`o{gE%w8*XJUtk}_5m6l74ALThyh82yGlIjOdmiN=df=iiE^zAs~ay@`pE zigADWjQiildBRRGSGE-Z+%j#(YfquHUc*U#KK8K>FbXiN)X_BXZw-aVm zk>{v4lhaBPII_U9p`f28OObn9N*diIs)33c+~}GzmiOrDyZI&8*nh9v`BuZQJn;|= zm=uJ=lBdB&=xP{3g5Kr z58$*9IQyhk32O$YZ;No~*jsz+6+22&%Achf6pWj{RCSn4Iq^cFP*D=(iOYz5TMPiO z_OS`ZqR5<>%|)Uu0GV$;*9-S@@zYoD@UX)|H6F@X=TnHWI=ld>5^DH?szM=? z|H@*m#YE?2-2msD(j~QhT4$S<&o|aygcv@?(~5p5^6FrK-SvzE;g8F9O(!R!+C2_y zfI#mn@Wv)X8BiA}wuO~VQqkpMlO5soW+s$vEK#B#YI zYT**_2t=Y6z{fy`j%h1h7qP%&*gaV|)PlcFqgpc*^EAci>77Xc`!mi!;X#2xj{0PY zHv!PEkDB_}AJ;#+GBIe?Uo(I?%^^kNna=A)WTnZ)#}tCpd6b`K+Q|t47mE$MBgb|= zDRZdX)M%_q96mk3NwLnnGK+_^C@-mYN-L-XX<6)HzyYj|_XdJMXSR+>Brmdc9rO|D z875FjQiszE9Bf$_v$&ubC2`V{B3KY-`knR|7{ok+NN$S7ZJ?j=kX1H8he=RCQcKQb zIKM*~bfS_uNhXkp$sD5JcQ|}LSF_8;y&03`|$P9vf%c0}Sy|^7SC6PeR zo4isQ7!DC@q(}mr4xCaXHADUhKa)!rxjS)r#Q3;*z<bQwxLT-rkpJoys&dcrt{D*%?Z8016{X5f(4&gqdq9qldBt zFBi=jvFcbNqKFRq$ZHJ?sdN!Tos*Uh%=a!S8!_&W5W@3J5HG44`P1JYo+tHt!C`$qh++5Jw{ia0PJJ25 z$HAo-Fz7g2u0wm=c|#@L7)HiYvs@~2aldzp7MV6(nj|Cdd4D#Gf-Ilfv+HCv$)Knm zogRabCsn8?4#Wh11@spvzsIQ>=&u3AR>DEoDUQZ*`5KTCJoZ_gNxuufEx#+?qm0FA zeqP(=esA`)s#&diXbPvpvmGe3xut}<1Exmxoca9f7fIP(3IrmMq)OOJ!AOHB(K@-g z=hMqM{6UT&_l@6jhd2`7{q9%-?|SwEXz%SV)?$F>xOF76FftSrHvbe@xM?!jEKCT! zRj*RU+cr+hA0+#|$FMFn<%E$}@gPxYmTWIR_aZpH6dkcq!Lqyu{LhhS&Cm7tqEtV# zt%Hl%fMz>9V0=G9g^7U)@T~hU<8)ysnH6vquDNoKRXnc8aGrEty{0?6SE4^?s?Aml zM-wdOx(RT{s~+-R5Z3u((Ob6^WMC^q1R#X*{KlFaDKCCS;Cw+E;4#$aJ_mH$Kit0_ zQnztplw#LK3>f=dvtodSAcZ)23q=PUEW`B;&(3_VXAlDxlMi}XHnkG%sw#+5=% zX<1GNG$X7`F6TwoMy6C<7(zh^v?|skD0zyNnYs8i1d|WI{01os@fQZ4Z^1~(joXBS zl)q#O)qtI|uFz0sVmUurc~FxyG{ktZnU-1Ifd0d&0sj(pko zsYX3uCM_W@|8U}y-(MU5S|p`fR$PGiHud6%cm;m5+Mq#K(T(#7ITM6XdR z7wrw+DT8Vfb&UTie|OW+v7sC}uGfh)cm2gbk#8?jnw%sve{CyFRLTE$(LeG9w==ZI zYNp2rKm3u!q6bcd&WP{1?s7;9GBFaVx<9UYF+nR8^n}=0ed}id_!}RF`?Ckl!FttS zF2VpnYg%+iGCFW~p%DgJDHW!G)e%Y#6`)bVML8H-ftz_%OIip{hN72YjOykFo_dP~ zRyEvs;H-_wLd6S4W;4w^8WCh}^qzXad<|~y_3tbw_tD2OpnX2Vb_L=btd=Q8@Voch zGkHrOU|$sh3FrQh#~--wbWhj+Zw$u|)eemsspy2LWt2W!gu^);{);?v@IX=SGGN8w z^BnE)H=;TFu4gX6^WiT?2O=0K9Z8bLR@f)nE|l2T{mb(Z(-GdO&p(&0n5@%m$?t-l z-*4H=lCLm&XS*MrTmdV2s=H(0xlO_0#KWS)I=a9~-L)K%t)zo6Ct>ezg_-!z{@?F8 z1lR0n3osp--C!(m$It#U!8`Gto0e1ddtiWmpP4T#)s1ZZCSr;TaPTklPO(a#n?e(2 z@8V;`9<3&lx>R7qmNSJ}V8qAEHF>n-3+6XS>>mnF#b3$L*_Ufos{7;UG$|JrzDza} zmZWk|R@7VP7it-uAI}Z>w6wkcN<>mln!Vg4{)UIIn$d+E?I+lNme|)yj%_Phs@6Q3 zxBF`Y_ImpQDm`Q+*Bs{`d^6@hJFe?eC1&7AIiY8QePjge+u2E)`=?=OxoQWr=1j=fBHaJOjZ+>Ao% zAVg6+<~sEsOmLmn(Eg%nir|eyf?oY7_{vXtwZ8~ln<}S?*MpUFVxXmDY8MWpqUJl` zOLfrgRz^iv{m>vZj^*CuWy$r=|J3-x5HKh;{T-BRgv9d$KtTbM?j{PvFLx~QwZx$z zOpzs)MBCp`sLV_Z=V!ehIP;O`S9@NhJez`o-#XJXDDj#@qXZxFCi=!mkxbd7f#VT_ zu>yQ67GIL>ruTXoEHYY(ccM-bWT~EhJK2{Jxdbs#=~iybDz&T_1Q7z zOvtPkh{0h{&k0t;fC2ZDl|o)hg0TLPq_Pk+uzXQdqx^T>V@H8Tk!@5JC|T~I@tEwI z+lR)0*5DfNleMpEiy>4Jh_2PoDSKuEtdZ2mO$7C+S#F{MtzIeev}yPaT3hw8K4+@~ zsOTdpGMO}KI>6GQT;#x$S)l!EjpR@9d~~}o3y~jby{n2M-)LPGxqXrFVF#%5BMuCP zdCUWd5?fsd>Xv6i%ue{i7bw0wg9&n>gSc(6>^|;yC%7!Mr08bi3r_p-2bUa%cYkJ| zMewO3BX@9@cSz`4eV(O-84-3TkM4|Nk6bI{J`gy6apqspd?ph{IX)sjVA1s|bjfhR zSJXW~j@Pa^)=B4gW#h9T^v~7lzLoXV$M~?*-HZ>xwxjcP%@G_EE|=^A*rn3hjh~w1 zxl}FzKSV@r~n(+^mEu8VhYyEh!gS zBSkC9prIDn-WsRxUxwrR5x`Bql>ZP3NkQQhfgnRATVH*5DiLLzLI%D80W~Iu_P9Oh zwC%!YnOKruL`RNe+p>tN?mCtn>K)*ZHI6J+5QxMHTPzbnx#ks;hzaWUui1dnpcNPi z8!q;1wyVs~EI^;Pt;h)}-ziB^J9m#OXC^Q3a;LH`N*6=Hpb!u(pY0IC1+l-HZDee> ztmm)orL#JCzm1uYR4tZe2V6JwsF$d5u`MroE@-lT#9-b7$id1T zNUp3c<#sHBp>9}6HWpyd)EaVL?Viz6N<^|fjuv%yZU!K5CR}8k5FPjx_G-b$^0jsg zk^F$Pgk9atiHgLEVA|V~r<~QP|K+co_*=x9)!joEgX=1mXt_-{;0A%9?r!jI+}$8k zhl9WBN^+osnj2-cZlv9|iT^FJwM_DIY4P3F-;)qFbeYC=EWJP0Qw9Piy}$zpw{*BZ zGr96e#@i2z*KBL^N8=X3=6i)M7M)ott)_@-#Ptt za|TYft8owH|0&Seo3#?|W9!t(j*L`kAGvt$XxZj20+WgMro){oai=1Y-R9f%;+T|z zUq>#}Djt@y%&S7AK`Ai7rmyz*f$2_Fu8|#H0zZs?+SJI2yKd_Vp)FgNC*QocB)Lw= zlO>!>Krol4d%U`oYoaF7;tbgf$XSx3>Kgsq;aHj%ky7OpN&@SMcTi8!+q^qS9DDrp zngE&Dm~Isz8(tfrODv=f5DQ1Y!?kZqlDW5oAS=Cc3Bm+(MmRUqv(3*G{!@jVK|#iQ zbkC#(3t|9Lp+P|;gtE@#hxsJ6A*F@}VG24uCXHy9pOp~o!UJT!WSRWlas5F1FjObh z$Exn62TQK?{sR>4(dBcXywRwL3N0zamZDGO0c_&K9d$UiqJUJoziX{fT6HGJHroIpP&(0ulx zMV>7vY~g+KqT1)+Sgy5J2}sI;$uw z@~7iXWXSiAK7$M`t_aV9+!iH>)dO(>E~68JV;JK?wdw(!6T79L23$oZ$unJ$b;GBl zze~lvhaL6lJ~ro;r|iNS6rCR8l>4mZY!o(xY-nZvw*w7LY-KH!@($gJSQlBu(?UL3 zc%YF}{Y$~nLA$NnQ*Mq6%o>NvB5JPuRr2a~cRUS22FYargcr_LiH2#bUL750+qdDB zLbzr1fL=k2656f2WB0OfIfaB={LAG5&OZ^ItlCls($l#s1T>?eYFfRXt3hlhI6)FR z)k;cI4UHdb17DLk@gBsvF zs~BBv5G?x+K0UGpH3zcbCp;2;6}2I;)=18&_@fQWC?rjl8;PPdz`kFkBrgaKao7Ug z)7#cok-rA#`y4-e`QcDj;|*F2`V*JtRY%;c2o@}I+sDFj@zMfx{VU964# z6ZTprv+z^)U*^3fYgp?uL|$_dVWF6a0BbB)R#u+hr;HAuCWl^jBr2;z&74$n*Wyv< zz-)h)?Id+QNkHR^Po~wTT3fF6HF0>2H@1K>o@E0a31`vwkmup)$^HtkNr>zqIw>>s zC2ht-Ha)OKw~x~}#A2;L9d|NZy zi=i!mXJkFqP=pVjBL75%z}`$!T?ad>3W2em?Lt1Jwjfj8TLjNyWScd9U>FJ7+?wU} z3a4YA(PxPQN5yl352(O--}+G#TE6W;U?G9Av-6LTpIKb9^~3S@Z>WFX@>02L~2imJhSL5^m`5C9Z7x z(Ib|3Y3}z?-4hAkiKSoKJiJe`(WgUUl_(-~7L!8!3X8At?Gm`z*B~L%Lba3EgWuWO z==JpD z>(eML*A+Wvg$6J?F#3x_)mt-3gQJeAp_+_{JFF=t(@^D4yR} zL8sUUOU=jBr00KO_5D?Wc3I&*Yj~~uU+m6Z?gwdm?dI4N8g1AeCyaVPY}xl3=u_B` z3*d|QxC6Vu*ll7jNE=<~Imts#{Bdl|Q}TC2uDDI#FcW%JvsK_ZVK01amMQeWEJ;PJ zChUJLLR^MRZZ_VWkZS!-mm9%hxrN3>X;o2fI`_=+@{15pud0k~r5Em;i{DD#U9T~G zpKAFN3aj72gT2u;a|p&LSQG-c#9?J=FS;xqHoK*ZUjUVcP(1J+dZwG3!{Rp5TPD77 zMvtqSw;$^1sXf6BeMcUTm0h*c#1X-`R5Rr7rGYe>e%0oYzK%QRVQ((Gz5hOjm8?6pDljZNmpcqY%`l}5ivfmHTQjACMy0YQ{2zslh|^{Kh-kd7SFMGCa3dX8AkerMVGy?8~ex zXf<)(O7Rs0Yrt7%GhuD);8uzR*9k|RmDV)u3E7Y#_(U>;dQalg54t)ga;ag)%$fM( zx5%@O;c#trBpPh16}yt+pdUO1XUK^mdX(J@mzNT$@WSR|2qqKJ1=*m4V|LHd%g^f) z^lEdowNYqcLp9C-q>D>|5qt8}*1l(Ob=NiYWCJBowg0tZdyE6Zh-u3^&4UX2G-vO;74uOgmVjXaa_w0#1MEP~5;qgKT-h&a~nl110((urS-8u$N@+vnl|v{jIe$ zrI99gNObkK(`6bRdVDdV&shLMHOn^;okGrceeq$&C&PGUDbJ_9TcvMAG~f2Q^Gan> zqX#MB1pA)0#;8kb%Zk|s)K2PZ4$nbQ?l{!ze5S%yg zEzY{8njhN$_j>T{05){PMTyDVM_b)MVd!39I^_ji*oC$l<)P99w>oWT;t0gM7h2F(+{ z(H9MN?EWxIx65WL1ilCo-pfow>$h-Ft$7N{o6Cjs^xil6X)O=QpOQS{|LANA>1T#4`{ME+7wb8jcWP@p+n4C!abiKW&4nAgDd( z$I}liPX*warVizzJ(Grf&ll_7b0e!eO%K_8XfJm-tk;qs0Q#S`po;nsVOIF;&XQEm zlQ8%Ck7!sajcni!S0*C^?cb-T$L&A2sSzfb5bya)D_W!s`B>XsNZJn<7?_snA(o=a zUObXi#rc-$jyoUY5x;(Wln5JmTkE;)+IaANpK*lKc1URW_s;Uo<^t!kNwVh0Z{vn8 zV{3_qUbuuC(03hRe#K&J#&{Nv#P7Aeckp!LyV}jd7?*$;SC({{l7q{JABvUc=O$I`=<`oMDq-2>x z0AIPrIm^@fX)fvdA6 zwJViKT{Q`vEf=nYvY{HbTuZ1c%9WVNxsjX$t#<{(i%mwq-7zG=mU6ACX>4eg)&8I( z^njZ+z+6=C7#8Is8IT)vI=F&0TNRYcP$+;IMP_hl-hwpN6khn`d|+( zl()@Td#kmRlEn$GsiDo^!i`mw?&p8QjJWxnn>}4VuDzR05LsYN9{nx%&(;7hvaFjL zt6T9UP`|1-p1-!qpJe6ry*8)LBiR+Naz1X(nQg)VB9+x=uurei^3SPrdgB?NwJQ-Z ziOz?he5%?W%7Ob_`_>xl3bWBU0Hz(-kH@+)$>)k9ipZhU##LM==dCriO;jSSH1d!N zP(`*S;hTXCR`INuZ-W-hVF1AZQryewVL`Ns>U8ZCz;IQ9ETZkq^iI)*a&4<Mg)P zW?bIkt5!S)IUkZ;40BYY`%AdWxKz`ZNQ-=gf8BBbd}P+1B-)voNp6=rgB?5nz@NLI zfy7oD+|Ki%bed>VK^tXRFAN=%`t~SYFQkr0Z?4N!pQBMD5(u?#((^Q^M-TE|>s4Uc zjKk^bAM6i&iKRw>>HKi16cYy&j>n0SyH52|=wWu;$tUxOQH@HgVRN*66X}9MbY+$% zT?+=T;y?l;n0g58)AW5IJY`uDA@ zA)X#f)}^)U4}z`i&eZkzI_z9?o^g8&mj?F4Z@W)6Xco#s&{Q9bmqea&lcs0kK?qzN zo-Fpz3By!5RF~LInf8}`CkxtPht6QkF})xr- z`73|u73w1?{H%jjFNw#x3IwTiT7t-e42k3-&J@^hmEx)kT&NeZN(GhXo(Ie{?bFzS zP86Z+IYL?sIr8!>`SYqo%JO7U3WCGWb`y94bA`{vFgJn%VLwm63fNj3&y3rwFk;Ry zdndj<2O^D5%}Z`Saj(k6E%3(GtTO;2aH%U0Dnbh?db3#GxJaZ8>aHo!Z}-hhEabTU zp%bFNxKI_(%)sLW+op-^U<|qrmJrD^sYF+RA!D2dLDRc#Ux5~3p0c#=X8%yq4FOG^Fk;tV)yC?}qq=6vai`ck7Ox_M+^rvWK{mmS z{4&EfcEg#Ew9yo`F4OZzaOXu313n|7yu|AkkzQ8xkBUXs{RKNJoG&^DEh1@16hkt< zw;{4l7w3V0$Hlg*a^w5`Wohfh(q%eHXeNm8FNPlxs0AhThf9E&7cUo_I#39X2!zHJ zf_N8y?+BAhh>PH$ppiHj5Hsy@1Rz2N57E9p}omwR9Nv26T9m!bv1SLS& zE(9`s{EbovGY0c&fO4tN-8y1WOrW61HRG%=S<=!lm9VNq1VUy9zs0i-dw~`Dp(ZBC zBhn++MsV8Nf#*cL*ibsh=bQ&DBn^cO|Mvmz#@|*KS7xM{J8(E0ygL@-4SizR;#|n8 z7*Kd^?h=mz>kxN7gZ9#~#A?N2HSfxC(gvZDhpY#?v%rTpyH`Kn;qcmO6nOMVzPp|h zjin`sW2bfivJnM!g{0vmQezGO1b>=yo!pPKJ{wMrD=s-Zk-bw%HW%s(yzTrL4X^gE zf~5q91n1}xb#oh?ND<7n#VO{DmT$y@h0ahQM|?V12VM4McES%sZ&dEW{~J4ZLkKh9 z<|U==W^<=&0H2$5Un@kB_XRe#U#WxAx)R_?ZRTZiw?xC~MlhU#nI}TDuEx}7*Dlg< zW~HhSgJ(U6xXmT=C&;LeIO2jnu=z54e;B5&MaDn+2>V*hjRY3A`{Er*`ghRv)s=BT zZZ_Umn_Lyc=`Hi7J{LF1g@1W0kB5*|H6Y)X)}GPfwObBcZ~!qANwn24QsSk4(n=(XFLEP>Dfo;q zy@~di+(99HDbq(M7_6lNVmA7e=P82bXh@9Pexw6L8<{GK#-?_A{UR2IWQuCJkUfV< zk9oZ{;%x)6SNTDjwU8y+bD{dy#l9TNiCxi))Qgn^OMzYt+_v%<#L>2KqLhU*q32P0 z>D>@UFK!n1#62ydQ>a}&4FRK#{>?}r#_3-m1l+1hL@Xq<_B!U zs@mu`k7p`V&GUspehC|-d%U{hHAPnA-_uyM?TtiiVAo-E&z%KuF%8~xxFL0rM}=y_ zq_M}W@I@Jh>p;NmN10`|b3+KN1K1W{L^{7mBrjCqZiUm&t(3KldKd`HC!~WL58&QW zD{7~hILHT?yS`|=OF3$7P19s|na1L7+4yK@3ortk;hX2PO!D=O_O(cBpKYNK3FrRh zt|Q)J<(~k9T!7x<{zcn^#A#YHGCTYi_ZNDqKg#PkHQY%Febfi>;6+~X^wdR&U~#lH z_8}i!CufN*ZkGX*Tsy!puo=5JEmdsGo2eBx`#9j}GkEf87{|kkNqS{%5 zM#uqH%sjO3b~AYA?hz~{&LJOO;4BN03pN$4;hSr05LEmLTc;m4slkoB-EUxSEI~C% z-Hw(6U=P|9T{!hA6UwCBT$68URh!Qz#7p(P8ehW9^17FlU^JRGGvTOOE2rNj;kBUS zuve(1z)R*|>X#P-s4RkWd4i|4Z*SW5^rU1i38XGcf6L<}{MAUHjWG^q+r))0bIUtf z;a{Ktg1m>jD|TujxvC0Gn;3DvSRE)MhJMv=lzzC|4GSQIel&UTuV~MHEa(e|7x>V1 z3-849g$K8~8ahVefhP(Nb~A|Q<4o@jFQw$1LyAa~OnB$Lp|x#;ECdaSHuwlb$&NDR7|tJzQe?h5hdrftJ0NpflibI*{M7 zMtV40KPGAJ5zrJv!Pne74X2qzdU$;Y)aM^6);D~{)9&|txH<}N>xUCC=R29&JI=qs z=Hz;xg6j$<8XKeEKj=mE#K2im@xSpCcJhBUMeFx7P?y^F_tv^4nuIMaOLT^cI11Nbot?CYjK=m*U^ zhTgDvI}6S<4amZ3iI2v1AuMH5Ra>mPZ-k?cH=99p`s1ayK{=w(2L|o;uT{F7C_)0g z*HfpuFo=JzCm{++1y}{K!CuPsrB+pHaO8a7tqs^ZDTak zEfKlGyfMFQlg2#4M7g4AD3-z>m(x#rV;HjbX8{S+4A`WMKi!KijK0pfoHZlNkYJGV zh0HaoJ6d0N0C?1I!J=>#gr-Ee8k2mQl{wd`TD7SMeG0bCXDbvg-qqxt1HdaDwqfrw z5@3H}!lxj*rs-$J!$dx1XkQFr!3bYc@w(qz*fFeKRfM)nyiB7*&p~vYy1-PfIaQ2z zL%?Ai@W)P*2&}HkZ;YKmljwPN!AO7f*~A0t_6GZ@X3yZ&m!}#O{uW*#v{r6S%S<_*;kMWFz3%6epR{x0zTW@+ggq49ZMfI z1huu5hqBR3J5iurKZO7Gxba+2vRSe15-G9By73?g$)F=evhx$R=i_0e6B|%~{Q8{d z9?88z%bel15$twP-M(p)#$w|Kx1m8&h!Anz-+qyaNopUD=y1M8X#VbK18!NA z&e0!RKm?mvp*MnhA_8?G8==;OC>eb@ zgrb(WrF)dmquunU`x=0M0r{y)+S1lVb8*S3*rZpR@-L6V^m(ndK$>l&isLcyH_08( z3G`4BDiB5a6!UwOCgRc5fxI;~h5gCr5??N>7e3}^a5@{G4*!O94y{2D&kZ_a9YntT zZCc0}ZXAn3xK{I%^^+~=FD)|}n4p%R0WiYP zXha=!zlpzKGgmNE+1rf{O@aiWrBEN91;ILqLKk;=qxE;RHY4LdXdxWM}g&qrWXJ$ zL7Fg_>`Fw(MFNtrqguz)_IjJ8Kzlfq*gt&manm+5N#x232wn*_!KsxYKFwL1xX(wl zc$OQAV-M+jDn4dT?}VA)f3qFw#C2H+R2N#vGvbPB<2YVE8br8k`VH@lS+m8|GW7Wk zNC-xLVjHZD2nT06R?{F2%>v&}PF6l=c;Tk72ltu0s+B05TqSZ=WFrH5bu2QX;2>S7gQ>!aOet0JRx$xSyWKJi~9$T#_@GMln5~Q1+ z33u0?7pO^k7SOmACBCTf*fQozy?AvqG*nAib~_r#2A?6lQuVOGJPIqy@%T0TeVNum zdUo#Nn16hMuHHq@)TUWB=x)elD~XLU-qH4E=0IhJhco!R?^DlYD~p}$uLC zXT%ic&Xdqv=kxp$S$={tCSzVvSu3nJB<(Os7YBxV-NUJQt3xcdCu#E3|A`AwZIVzj z!r?ZxPRRRPIoGXmYrGHYg&3_vt+WFRh-vtJq(+TCnARj2D!cynhVf&!w~*TfF*!e2 zT^R@wnB}48UlGSG_m7U2spuuwf==q!=XrB;6s_#u{&>`Up=WVd4aL&8zrK+6B-zho zzWK{=ZUMS7_WOze504(Dybsu%FV{!I8>Wtf!ql34-nDr&6>sh%HZ{?Z1P74>lfy+_wFDKl;m~7i0pk}6tl^|b)Kk92*LFXi;pqPMl^F&69&VFeuD@u9 zlrVqp)D3qo_DSB4x)wKkv+V>$l*5P|0t}BJ z^Mif-{M<}Y6iumNwJEXZlYlIpdYGiNUzU>1_DwMaM!Qz?YRM>uCvGRQ!X05Mh|*ur zV0wz=kgyK^L7gd4gd+w-=C{R!fdqFj({b%B?o**BbA}ZofvpL2ZdAIF1IkzX*H6MF zX;)xetP+Z0oiPmrU4ZyBb0(LQt2}hl%9iBi=Yo}kT~1-Doa?EneCJFewQC!jW-d|kjlQ|jH7h(cilCuM78363yUe41 zeNxl!4{f52K`?_}=G$Zh62~UT_iA^Vt;3fa0w_%WIfzuOs=w#u!l)9$P5j*$9?hX} ziM@=;hSL`;oHLLrR!r{jFr@aHs*|`2)H3}&TFzD&W}QtHSrkC;1APF@VkI7)GHG_X z+X&Nz=B)GAc&(>D$!Tfs!zsbP3e?PafSMGY28&ZoXL&wqGYuJ`V9M-6Ly>j{R+6ex zIMBCivXFaKtI&+_iAcOpL9jP5CPt!*DdpyEKGlVKeli--~EtP}1SGInMO!B5W;HWJAb|muLDx>g@yoBALo|h$qee`I%o{Y#@rI{5F zuma^*C{zmY<9nWwVtytZ)-jYq&{c3y#igH<_vNSezLzq-1Z8d$%4Ic%JZY#)8${?_ zA$SFs$^lUvHXSq^1NOHTZ2^4%yPaRGg7@psxc+SK#K^;Rg{R5ah8hu*VtLB*N=0ES9NC-1azT|* zy*LkBEx5!lINfzWv41}>mg9Pu#993mSlGBX+(MmRXCyy@5Cfp1A&RoLkejVZt1L1O z+*@)+wBMStPLm{u^|&xb+>ynQQJgr=4T`^eRSK#Hr2-uFFm;w|9QV#esS}&1n{AJb zq{&Ptqj1{^_nu%7i92km6R+#ECPEn3!NL3M^u4n#jgiB}8s^C#Ar1wmaIYmlw$>n8#qYUCU#59!#x*< zfM73^j1H>2gobm^824kY%}46F%^p2^p1HRa=LZa@(j>?weRo$KiUvn`$&YpE-p9r< zNTpQm7;dg~#6UB9*DvCH8!~Z?kta<^L1v|Y#hO-0nV0?Nd;M|SKc~jk!Dt%O5P`e5 zwr`HNFN-ydZ=(bl>w0#+&c+vH}PB+2EHUZrL`)t8UL$1kc zFR>Brra~SFJ6-UR^NWgz0_3($M4y%~t--4^d_LGi1!uKx;)^tJaPsRxQuVBUNL8s@D+4zI5n_<}UORVF(oLDH4`#?>D`g9GhOglVxe z{_Ufm%iJFBp8bEq2WJcl&Qxp~_*-aJbt6452=7%=#4Nr6zo ztA^gLq6Ycwx2613E0UV*9s=^#=@op^=2_qF_WS+nFJrrdRL=9c?#b?Dn+rxznGrr{D!3eQ zhPpayTc1AGr-B1Bo0wsvl^a6Nn=8!@Gt>Jk=g}Tgy~bTrb({!6oguWUP6l*dpZRwz zja<>Km{>$RYD1u`5k9!%b3KSUTa23N_~TW=NxjKpvpIlc z*2ih`Oaw!aK;Z2z0;6cOCOe6dn)Zv}M&FxSp!WGEa4QN0AD>7X4tYvI>r8yjyC4dwWRaqNpAKX2E%Xr0zH9E|nyXjc zUgM~)Zn5dIdX*T#V{AA1rmlPcy}VPaCkE4nXG{Gxy7g*`#Hh@vvzHBC;gXUCE(aj|5wc#-GbY?=5# zz<21L(nWjiW^kI68En;smz=Ze!j&d6D=W3sY0S#WNMFHsC=exeP77;s!m4EUnryFb zO?dJ9!jn9e{VgfGGCt14J{3$kf^R1)feRz2zz0p2OT_X|EA@b9hGWfU7Ad`PsTMK?tolFD*GnLb zAfUj@qkGuAZ@oZBp1S@I0C7N$zt6FrtS1#N8dveAuzsK!63MP7a1M9R_x}7_uTlZe zNaCpTyT_tAkCU5gVY}- zUj@GHiMUtiyU6POp$nx2o||Oa2Xg29(zpF}J1zl8hm%4+ndq|N}a zrMP_Mo#RTqr0d`J<_cBPnS45t$mKq=%gF15E`(|P*#HZDC0g?P*E`8EAEpM0L@~R1z)Q>)a`iq< zg8j`UH_IqWH~i2#`q*aeauOZ3B6)J1Re9!fNu~y@(-b*oySPdpx{9VlX|Z`bOb>_I zor7ysc^)eXPd>iJ30`I}IKdOKtNI;dQD5TOw*SDgY=;Di1msrrmN?15PbV#&Y;Bkp$H0q`qUbQVu!u zu4}zsHTQ8PX{!M25vLfwON{j4C{pGwxVU1KF*6)|eVqu03+~^(zD{g!jx+^2r-E1^ zzC$eTOFYZsoSTS5-&|83)bk2g%_CS%GIxqYgIYYI^j%>jhpVn-9$?vAvJ0PU*Ml@x zFZ5TEPW|X52nSI$I^HP03#7T~GAhaUc`nYEOyTLbD^$&u?=p?9zO5rVk^1Bf*x0_Q znmGOD9pGvgilkj=cV8`=ACUS{OE{Qo%$jhOc^iW{{M1}-@X`4h5YB>*1#!z+EAQp3 z+xd7|aa{_V>!jQO=oMpg*Cv_%;2!`+$73$u0n%4)^tEi^nLl4P8@!F>fVoa%b+>k% z&e|(Te$=i!W9is}nC&55En6>E%ArdI#J8XBCFm!TbK;nOD^L>6~R$faN3C1RuO8&ZVIbo~!=A$%2=jf~YH8yGb+kbr1OovNk5%J#e zT_9adkoq_C^PgtW08gy_vy(r#{8TrahD2_!y!Jq5z^z`~l0r;=dRU96LcRke6Tr@2 z%G%1Kq(qZ{y?bzZRUWfL@!zlAE_*eW{p4q>&pl=v2<_$H{nuhXo;Qv< zyHac1CeC|vKE%`P4#2RpXJ2c5)W5i;87b-XffWaux1UfBQ|WniHg$0|Q2XaU%*~t3 zAMU1r!D+W+U2k=)pIz(ltCaZ7I%^KeA|!`wgq*`jABvMB$2p=FUu!ru z{IYk=7cArbtck}9Y(4475tHy21;-{yI8UM_xwjwy<7YeVYewo^Wa;2-(qShIUI{aw zz8kY$Bu~OX$$YrPow~5DTMTdX&KKDio%W07waeRIyt?iy7L)p<9BppnZQ}NlKFJpP z*~D--uT5qi2>UA0U9d+SQfOZ{_Y1UW?Z3bF%)J#PC}NOtP-Ux0KL7SVdB6GXZ+^8} zo2b_ST%-o`iZiwUx^;a0s7Aff{mo#|@BKf2zfi}n&wINEPo8Q~jo0NZCVdR-z24s~ zt-Ovh?4Q{`&*bI<&8hwC8YzIQ_=S1$Tmw(-Pu|9@oyfgcc=@yEZ}AQ>4^En61SQUc zRgP#y(AxptWEphlWu|+RlouB*pKDiv$D5tEiu3hThksXj_BQVj^T4D*8EogQSX`As zs4JZmegDMDcYpEE5|@RhaJ|orY+F}p7hHwG`+Pr1_bivWnF*a94)e3QXYs&*Eh7!# z5EQGmvH^MnyNLl|#487QZ+dl3-Yg?}^csE4R}OgP;y<{-y8t9Oal*dYYFz0RCuwgJ zX#nATe8#7%8oNroMi^HShO(7!&rIGsuOPW;x`7MurIO$r+}M6jfAa5zRxS!zH9GF) zzIB9G-s=`Vvi|&w<%elqM{76iWYY}Vdl%Es=KtuY74h_~(hXd|X0EZ=wTWh1y!HFU z+%IWxC7e2`VXm`k28GnPs``05|C7_eZ+`ya^m8fp?bJo7@@zeo=q_3_HB=6=*E{(s zvgKxUYh*LU%knmn02K14b}HHLzF6WzRFaBkG(YPG+Sr-eM{_OiXbu43S6ISq;Q&`vHa^V`Xzkb;}0 zNGrR&X4mX_2InnLqg-4URBSD89iakp^~ra`Zf8X!Lr;jR9ZFv!lb#a8Veg_BZ5Fcv zaBf}`+RS%v`qCbU<)EODIV_$r zum;9Cyw14rDK0u)5NH(0&|Fr;uA40)l0&V*@s`<{E=Xa7wIJ!M-+dieKMKY0)b^bA z-i=Sib%niqv%pI7jFVVK*6V71=3RmHcR- zE<+p3znK&$B>48Sz7Z=KbiS?QDyoj{Q??KWklKyYW4DE|qhwLz!%OHt{Ey*{!QUmu zjoZ6n4r#Vp!Of?e7!Q7RlX`(}*8>HP=iB%Jj*!)Gw-~b>tesfk0#PHI;eQWa ze>4){LhkD*SDVDn(F+f6oq?bW(b5>Cp;LX zu~n{n9x)HWj2vDmJLdcMlU%&9^K?eGNt#b{{bVFGT@XfxG2Y4Bt+`2a0as}>vR;3B zd34YUE_^(V6pq()Xb2o=5N2Lz3Dr@niFmWPxlH76tt-_1P1fgSsa+V`CW#L(&NCz3 z^`cts#Lk<+MJ5!>WO2q;R-okiSy++l*#%eCtBkDkE9m!I4N*mk zvW3xLK|nn+yIfX!qhdCN>m@+Du9lTQn#pk(3VKcEPw6M#WXPs_kv<%>em=^! zQIRW1A_!;WUfG5lv_yxmlNG!9Ifd!U(!p?{i~Ib%1GEFH*# zk~(_8=Hu)R@clOpD&CtP!jMEiHJsi~*Ip}ASzy9_48Qr~fGBHRd6Oc${_A;koJVk+Rd!V&3 z2Jjqlb?Sl3j8Su}^_PUo(y@DQ{Fim?LVo$n6WQ!;F<536#W~uW$~EofV?=Qra7(5V z!_Eo5eUKy3PbEU%_JHpGnPUJXJ`UizP&4T2>gIH6)a-6rSlsaCUsx&}k!d=tK611majBS63EK zq{LN=e+6|3ROx1bDcgC#lo%$uZ}3DJe^wA<&&_6YZNJS_GwR`RNbaX61b$%c54)i- z46@%@`;^|Gul{wUV9~J}Tvt^EkQ^ij!~RJ4hQG;AN#&s5&X+l$xaeILtI3#$ z=lYF}{Lvyf==DjRx3Pj!%V$4|IS+mEuk(8y2+iQ>%dS@x1YU&;VbG=Qrv`~W{nT)n zgnr-OO*2dkV3x6&!_4VQLG3sXrx%1P&rA_j{*O@a4q%-GdoD|Qpx$|#5j%Z<9`IZOIsuaJT9&n}NCJ!6VjT#QR zu9CtjSwRqRz{h8wXLCJX6BNI*XnFYHUw0JlnjfsRMwTstX8C|@_sl-~_$+(+%5Q+t zNQ8r+K*^$plCNftmqwVMv#@@XgAr+eZz;5a-$lr4g)Jf6I>ivAPfe+^)XF&^`f+(ZtQ z1TERm_fr>zWQuGrGl1-A(?lgwwaep}Z(=i#KP`OOlvf8=QN{L8gM(D={J_`IGqPf{ zzj#iSfJ#{9C3WmW+(539!)2FDR`%XcCh3t$rjp5Ia(K}Rh@#3P?ex}HdnAAJpDxN) z;MtsJ&a$VYd~a8NTA<%%PUE>sBSfjSBCZ(oz^snzDYZxrqyRh;BobY3NQxAscM6o9 ziCbp09xi;->ju|GYxuNu1<>?cuTQ>x5p6|lU;Q*kk^l!e`DoXbDAyI`GNDsl6T1*M zkgM0_IgO|^g1YDcbQmND$;6Oh`>goV6Kf1Z9#x^#*3Z|U%ESrLIM_}DJR?kK3oCI{ z*kO477gEnSDU}9=DFdEC#~ERp$&Kt3jfVCh#eltJ5=fHBc2B+Qtb1zBecHl46Rz~| zz@Ui?V^yWKH|+LK(Nyd&T2(Wv_?ix3ZFRgm+(=f-;S$%f1WdV~V))S{?J6_lGhFn_ zaCyC|fyw0;F3GnDD+#0?8ASCeouG5_+*Pu)$9|Vr;mE(y$9bO7;f^9Mo7zCG(1*)Z zN7b=?AT`N~I#BAQr;MB-SY;|2CDB)G{j|DlbSz=D&F#({NIeQcS4;aT zNcXeT$BhdH%T~!j$7}PZaWg-*0W5}tWG9p8lS(G~{iL-|i)z&GDC8}DH*2Xz9Udq{ z;ZjvT-;EN+M^((@eSkt&39OPQ;-Jc=ar0T+t^Iyxm>T%)z{(d%5BIaqe%GFk>hhQQ zlb`+JE78( z+afy{l1iuh>29$p5p|GS!T**U?EZ3YCd;+69abM5YP+a4yTmRglh7aj>T5J%C3X0w zDzuP1?+B+6?+-VP8&8U$_N3URMWf;!?dK9HhJy>vPg~C*Y_avU^m~bZ|K;=RPi7vY zA}eF@b5oJ(NFi2nO8dD1eJ}V6AQaEpUJ~jlG@@i+{+H2%8vt_iStd{awnlBvi3<_DH zQSsu`*X`iuawtWV=Y*^Vv?Dx^ z8>B>hk=-kESqxEe_TfhI7W>1|En@_qXA8;XZq~1ZiW<$8Pi`YA+nv5SrO7S{lfp$; z?C=w{1otUqiWN^wJ=s<&ka42JMsjm|S%K72OEglHt^Is*&^~*q6yt?ayFl$WD??Vq z0&x{`ocf6C9=73y5nwcKWK>tdh#T~fkGX4ZzCUzP8eKKOu`1_Z=L_j{*8a+95vIVn z0AC&DPnx$Th*Orz0*Q2|AE|lR_$I8jmSM*ys;0tGi#WWzdAx1OkK>WjM-{Z8@>yX7hMEIh2U#qPE-lm!Edp zO%u9pLVwIxpXY`<{fxY8^)vF^DC9)fdDL-TMb%I)`&5te?Y&I*G>hj{qm&3@&A2JN zjU2iP>fyfu)=B9!@Esg))DK(9qmM|C=-yg|TqtkT;-hABwI^w=T8v&|5LUjJC18(k zCZ8!Cwmo^dn^VjK^H%zB6uPY*W>52Z$Tz81jfkSY@~8VLpz{lEEeCqaURIGPEq-TL z)vHEo6^drC*Xh`__BvTOYn572qnlU|H;1>~9|B0{CD_GmC+mbW?lgcD?G!-f=uRRU zi^R{BQ5&dflcfTFkv7=hmI~RlM}P?ri@Ppu4sR!i(N**uXj*MfY`@}(uqm+8BFug6 zw#2;|2FFUzl%PmqDl`zbm2Y{@VVnM&y})V$!U?Nv;^4|=a2xti3lGW=STDl#K4EZ? zuVJRYfVhvH0CPx1zsW2dM}f0d){SnQ^<;qW&3v?|B0F7cH~eBhj3H{3bde{ z=RCtHM&DN>*72w4Xwt24z_uLEbMQSQq#&H|W1R(oUurqt#Ah>Ff`x76vjHMpon!an zJ#s5@sBkE?gbAn5YwFde&*5p%G!JnhH7?Y^D+L4%;o#wfo@<)`TAmRw*NaEFNavul zi{(${^~CPQd*XKVp{@{kgoy^x-rw`;JcK?8qjd_5_%lEnFv25Dzwq&CRKS?oreG4F z>ewB;JYHPUa{t`Yx;&M6Ej6xN8~6FRcjlTjX|&W@YR#85L>;*7HjHG_11q3K*f<9n zH+R23Zk~@ljRGk!$7+;@{V-0C$)|?7+2Q>tongEgl;a-m)31|S3LI!uJjZi9>Qv*z z?fc{Q{D*akl1yFWhumAaD$-?=`EeP7B|j2lLSb|~BV@%JrHRYJb6#o z1octL(uN_W#&u;hI$(m!BptC?wtYZX0&paPxeh%sx6dubQ^Lf*iU(9Yho^IO!}r2` z<5$C3fE22RE9ZGWUqAcn*!wJu*4$;V++>sO9jaHK^)j81D?O z3XK}iXk4h*N2X59-E#vu46VzVM}+6tP2u1<9$|kOrmysUThEhUz&O)Yjm}64_?(5m z5*HL&no&Nti&BW#n|L4Ga9v*D(J;buOjsf%^re1r_ZlKwTC@T#$g;Jce$#fUa*20v z_F-x6FPr=^b36M(1D?QXPzOC#MjxX7nv>_Z@;Pf03F2IH*{?V{NTkSg^NZWh&3B`X z${b?umYeixpSxBrXo>KIk;h6h?kN)TRXNLnxJRWjD#n}sHMk1gnkX0dQ$X^)!h$j3 zaOn?IRmySWx_jX!`f%aS3IHu@8i7GfA7ph3UFZH(iW?=|OsTQ3?)h)A_BttmBcyrKsjd!(pxxj;@ zu01|(FTN}>XyM^$bQPHR85@oJOR)`^lFuQI-4Zjswt zmnE30<9UFbUK<5GNUlJnffZ5vWQQ}-oM#5m`DT=)7>n43aC`bt%gW(P&j{x!4Kr$s ziLROl%HnfpZc7^2R^bf%Hi06_4Vu<|ga%*y^o|X;K8%L;>u*2SWfSUTdI>n9Ho+w%AQd?@wE#z2O8nFk*ay{*jOD`q0vY! zQ7$-siM&0^b&YvoZf#xONdk#vyH)a^xrB`{McqYE{+Ms>$9;IkoLn`EIa1TYw)pzU zpAMF+r}|>+`A)bsIqWAAFg)M;dE+>g2P43x=6v?SVECzhA6c<0=j6Jho+GkiHysQP zytsDEE_UPG5w{|TuEO}c?E7ANxLZe5pR|{R#O~337^W*nH%CdX2nyLk_5~QUd-KNKDoUn^ zh4FT{t##Rz_;#@!Fx4DJfh!70;JkH9A9KY?EPqNtO-l`qm+nI}uKdPQq+~Q>I@XG} z!>#QPwGJDwYUR2rOoRq%^n2;yZ(rTQgtNCrB84!zp^wP7XQIaIdN;>8-Gp6qfGX8c zq+@Y>J8mzY0@_tlOORU2YF$Z$sX%f#O!*Jk0C-BujDTYm(3;67UuoW;kUDZg%IR1k zfW*OLE-({weQr|@A#6%51Q?`nJW(Q*Nc`4oJX~jlNuiUL813S^Uva$-hhZ_!q%5Zf z8jXv@w(t<4Yar$=xV=0FphlS1;9!Hmwt+sK?u&=70CtIX3&59)o{q$9TAVKHI4v2? z|B=Vx13rWa+p$7?J8n}SMyjV``?l{G$MbCh_Q>Ge>qT0&-E%LQw^3fhYTVGEX}e}qtLjX zsku@}f%8ncfGdR&(eAM-1xBmbm2)$0RSvbpXsFnh-NY+fVY6bV!9)5tuKHjZuCkM5 zd*SQVq+gTPPaAU7(hiVeuh|N=iCsB2KeRa$DPX$TwyJ{JB%=Qmtt;o7MoGKrZ{ zr09tlPKj65n|`GP!uA1i&Hc@}Rek6xjpkS}mBpgb_d8JWim1v&prYk=x-sm`>)gd} zBZ&rPMd&J(!xAmVVj$y6#WY2i4 za1OKWfQB&IjYEAmBNyk0` zp3S{o%r&^Rx=~`KgFW9Wia)ynm)pH$G;^vbZ3ua zDgxiaZ{Er2J|1paAC980NGhK38;sj?UJ*XKFJ|Wi+yEK=Z^r(5VHP!>in#{2zAn4C zHHp@`3eS*u&egBEC&EgVWlxTl;ne~q8H-|cq%o^o-#+|rZyUIS{ozDcR86IUw*lR< z=U8A=yIj4wnu`vo6C674E3+ zHLx=OVQ*ceJiV)2`9?K_5vGFehkZ%oVJxe-3w;RSA}gPtK;VzHq2ylYzb`ODZex(a zNSLm$xztk|j&N+hXtiPo_m$k;x-2*;THr&pM8O8MaJx0FwAxXqs#p%^1eTQrX9LoJ zH=MiDhk#8xD88gTPMF$AZk!}@#j8rBlIM^=X_cTJySK04F7}5Zs)}R#o>OYsRx2D; zkGRhf+w&5(U~@Q~>wlYkRkj@76u!aspKb1saC5j5eHiY;91dUNlmeUmRy11iUh`q+ zFa8?hFxFddnN0Yl>*w2mcb^SNx0t(Im!;>NlmejGri2$=)=g`wH5!u}rPWkNsn#`k zyAypTy>gR>nf<$k+?^Z>ywctt-4nn}j=kyeS1C-@90Qb%)XIH()9P{=Ld+`eL=N3ByeBwolc&L`OKf;q zUFriJXkd!bM3bCcG-47hl*yiEM-)!BE+PY#@8Ap zWet$EOF6qM@@6l+C#bN^;(Eu&gxv1_aJp_-I-IsjP;Ax-J#$8~0lUMoW>esS2vZ*n z)Lm2+6Dr7sN^xDJfOtIQwYa_g;k<`hRdGtdpSe&&;CSU?;hP8Sbpz8&rb2=+S^`>X zKG_O;Tr__uENE(vOj=C z>J!h~kZq!?v^$x7CDCX+BJXE-#LeEgw0D}x69aeZ$5sFh5K>?&_I~(L z-BFXXD_0@43cHKUHU*jLt-1agtGcP!ya?-2V%F!5;>)gzlu85Pc-~RIkWT{i zx-aT%RQ8t}R+2_;_KA(VV$js;vu_B)F>yD<>v2c>LtHmp*~AkEJ`Z?5l>pA0|K3!; zy7z6s*Y8E4G5Q$5-N0@to6DW;4;4}vzyh%BB)~xoq?gg4`ErcZj)|M-XCPVx`KFFKllqpFT=?PpWNU;S|O{KbEI zhTPaj8;Y=%Qh0t!-d366Sp4FwuzBv#j~z8fm3Xk<@9=UghdY1Fv%CA;w`I0tyX5Mj z^;e=KnrgT}xNKal*amPn{=<C^a_xQiOHQQrD^8=od4cMq9zg3Vg2=#4GX=?kb)##svbf z_sC=}rXJtO>oFTOi42yho(xmTI+u=H@J0%yltL-#;afoy)AD_9I>J)BSZ z0l?q?3wNsmt)qq^;}G$-ocLC7U;D#w7OvGaa9H?0&xQB%siMs-AT&-U;l1&{ldPPW zn{!Wcs3aUg%Ybuo&`G}mp?+2#_$OO+{YG*ZrsG79xi^jX#Z0&-Ig|)z?=-MOpLV;w z*Wu`&rQwIW_w)HTXJ!Q+>F+pG;kLPRIaHy?t>V!M08@ciJ_H|!AHB1pE^ec6qkd*` z+_@}ToMU;s4emo9D$t@#+xI|#Z$n^(H3IZEf8tj(!`@Yd^U0>CqrF`Nl9(~~=ErV2 zJ~$4g7Jj30(1{*C+ZH)gx!P`N0y1`WCjl)RQ#cO&oi zc5Jnzxb%D3j?>;2?m-UaWH04{JPLd$`$(x>SJfpOUR4ialN5)iY^10L?^RO2A?Eps31vIL=J82OU6wcpL22EHvoBcbwH zw*Siwy_{1;^d6Wfqj2;jm+rh(ZRsQD9+CiECYkXFYacRs%dx9im|EqJb^s`DX4O=6 z9PJ6)bxMI5D?2TP(Q$gi?Qmc56gNsqG1$cu>m>v>{uiy!cB1DuF3hZuXd^(QK#oIs zQUC0)9A1x(ja*Z)Gv}t%Gsy)j-nXde02Nikcd^mo6P0D8gm4b&sLTCuT?oCXy@ zTy_{P2=ECr^F+-0+?yOKq;^3|Q>Kg^9CWbqWdB><#y@EuZ=w&xm?!ym-mYN#V32xo zKZj8{m=e>%x`}_#26KOP0cfZRgGM+|?6eDe&}rE(Z6C8iMy8_bG(_1i{aoN%faic+ z;TcE?TDc2KMO{#sxmbB@#H`D`=|dP@D$4lypBCQl^}(hD{!~Wa!y}L+T>}Q8VNLrnfCT6}X`h)NlI2F(FY(F26r{YPe z$K4NgR3NqdM2!xmEo`Z+X3Os5QmU2QU;Q#f1s)Zf=fe3co$BOZ(4l+!qP`ib;wv26 zZ&2|Z?}!$qz^4IdwdL=$@x1N^vvj@?Za7`J~jEn3?156_#FZb52U4eobct^(G6H%r;yr<=)i69D;_%BPzX={B-63=*3C9EKNRBza)@E+?!R{Kz#N20|gT}G3 z=;rdj&&By^u0x^yegX#lK_UUY^KaJNj@{{bPV)Y*fpQs{BJ(qU)YLICi@862=(^#$ z3|1o9-^rX(AUAkZNDmU(L2@{}_~k?OZ4VtT#=*OhH(gEsai_)yj2+jCs6Hpwh|9U( zb=iRD*t@AT{Z5dlFzA!~P4~kA9nP^H%(jeLC*3fnGvhZL-&S&ZNS`3i-CoIC%HcHy zM{B969%OcgbenMS%~591WjG*}O3|r4U|UZ225a%?CDWO&(?U5Ko+}YXi_J;o*hJn= z4zFt=x?$JypL|~+bJX(uLoc7rkRaWmPu5{Gv$B^HDL5BGx75^KvkL+ zxo3OX1#qC-c^!Clp4t>R<1Cx$Wjh_5ZIiW}<{*zx*=m6R zVYbY)T)aC`xVW`&V>o|+)GF$A90Ly`hiDy6{6fn^GW}eN4wODU&6md6_i_uGfK6DW z;-a?hQ@ci2jqgp|Ja>Cy+rvZ1A*1hKiZ*n~&~3MD^6ZsyJm9Yj`C;wN*Yz#FN=iJ3Q#S5pD&z=?F@Fd>`@Ep%6@24RB-sfh^uUY8G8J7x7_J;8u zagGPlhtZ;CFi_gdraOfqc}{%inbc3M1VCbU<9gm)htNn~=!jA%HoZOlee+Oq7&X%p zPKkW3&u-?d(8VggL`!=wN)q2vuQo7>97UcTEk|qeRMlcEh|@fnK13P~l*yAycA=2V zcJhUi)cDqiDPZVrlc_R7%>?&G*~5 zbU&A-|LZ@y(^I<}E>$V31^E-oRZ>tjdh&|cdb!(vu%rHL%aF2cwc*ui_tp9Nc`ujC z6ptLF_g!%Q{eS)!+kb7;_|I1=x*>6KwTT`KAChrk<-ykF$cOt}fa5slTH zj_nwCG|_V#RmA9=m6qeIL$`)@75Jq|b2F)>MBp)>g#HM_F846EVg=!4s^HV^^t-(- zfNxi73ilmwY)mExDy;#G9r(`xkWz~92pqv|XRs^HyLXWX+#iNW0gg@3I?O^IY%5O% z+)47dv4qgq$EXo-fem6&p%<&dCeDiu}Cg!yP#ED;uUKR-|H2C3-D<{;7v?5qRio8EoH?Q!ec) zR2p(S>m%NV2Va-NY!vA^;Ng{N?`Hdyt(GI|C`maD?AV{V&o=4GDAqUT6_G&}Z8mT~ zfrF_6SomYZWc2jo6T=6WLs!AEqT<=UIBFdQE&E3uatEh&6Sr2ei{bgDI}d!CH)QM; zjO6MVYB-(&JO?A1W%4aVs<*<|IZCxrE=U8%gt>_tz_z^xAOujw zG3duwwz45|XdKH5I*{I@x6|M0XVRZ=o=y&u43Zc9H!tll^tCa#t6l8VPf9TrafWcL z0Fq;ctAqnMa4d~*8WoQQ;7+n}rdY66ze7B%AA5wx3tG*SPNtWEU!V6f339m~BuO$H z_WQqVmzoc8R^ud@MbBOR>e2tr?6fScQ8kHHt_#>DEF4?`p720GHSCNw`w6a#o+`3> z=LGV>^j;+M2MsEGUi@>h!5wU^GQL{U;T#|jbc0qc&B z4`m;WzyUZ4EIa}7lK18+f6#XG&~iBOfOsBQ`P02jhcj~TC4gb#B7Knp9Ln$ziD1{T z2QD8u?gbV}kV}n<2X$05z!Vqh;gksQ5RPXk`yPAwLbu!;eY}!kD#%7ne!Bx6S`J}y zv5gbJ>7Je9*Ej#bzm4?Q_a~y*Z^?KM53?H<>;E?T}23}nP6`@_lOp%d$pSHc^@uB5#x?iBCA)4$C5~&1`Ozo2xbX)iy zM(Z#$XPeQZmWV(dBnBbS60aeS0>Jk$!m|NK9)E@7VPQ3&JeN|VrM&!p@026w!R2ro zDNKd)!myV}CWeVb5{5ga76vVK6{pmqzP;nYT&3M;96A~}RuY3oyEReX3-YCw?STolL`!(?KR1d_wO08y`+ z|1$xDy>BIK?ILv9@8^yrOAEgUfoD57o-qjU1O`LdBIDqIaKaX8a;lBpSO~9Z#JCk~ zjvNww!8WwXWx6>4hFE))t$&&TlDj4QUCPZH_nk8wS|`nVnB$<)R2|PTfZue2Qs5)- zjMTd7co;FQ1l7CZ)!rOA4CfVV%D%OC+I#a60E3j3OMOD`eE<@9DzLTR`+_Zni$Y0J z1`O1rJ)kRbO4tU#Z#rJRiqW{rIOXE!nDDKb?cKEHR`ce_ApqKSfqcGehxbzqQmN!9 zzLLncy(jP>LB6+VX`YN0&1M)&89$<>hlkcx&%ujYuz;|OyXT!Ja@+lY&Ug1~|c6xV*5#VX`B-KVf!N*7q7{rt2ljmrpQ-scm zKYrb!;Nq5Mg>8-;j&|D`Y%AAI_LB)13`q}3pYiM5PQUZISuX9LwaZ&H_TkFaGE4*x z7vxb}CeRhT6o8~`;gzkj_2fB9S5B;K78|CVHrw(=T*2nZ;piK6-I`Z`I@KjEr^t&x`TGIzzz0#bDA~pKK_@TJG%7yfz>jEm zIH|94x7=vnAUTXsk_Lz8oor|K%o=nvNs_$x9*N-)Y%6$W1_u`srj)xpchA%fqb3?PJ5_P$9LJ zMpwb{@~62}YU;_vPPRkacRY74#JnoY25PRS7zE7hj}XF(A4@|^2; zEVedwyFj0l=#Jh~UTndei9LW{bo0?+F<+SXX*BJ|I zpMy-*Y=+o=@gJ`OB7FW~&C10&k?B~foX>PCGyfzA?)(C|8G92q%DR$^sq6{|nn>K~l zI8U+3lP%dkP?Z8dJ{KQ4gmz7=lCUFJK05Yn9kFs=x=4Hc{TGtEjhA3B3F#A zmVFxs$3Ez^PiWbe1-=h^h@!f8l7Y6dZm9gaHhY*O3VO_js& z3(`nkv4g!+XRtGO$Z$ZXP{1>DQny`sVQZtSqaU3Nn=VNQ@D1P}xu?>-_z77bM&3 zel&loI}B4F{^YdW+(N5)1q-~}lWZ;t$f$&(UTu|Z&jp)ofgByOwA(?ky_cbFW7BXP zBadrQOmHlME(!z=o8$G`XgQpuEos4Sa)Itnq>{PJUNQTsGS$a@OH)B7EIawId6)#k zT_2%85!lP6GQC&6jqlsE_d2Kf!19~m8FWp-XbibTqXpi5b_=?K&6dM)(T{YR zlt~Oy!%LQPGLgv+24@FWb?WME&ewY*mha9+S&1T=`vy2Dpp)x&JJ2kb@^AuRfk7DI zJb4Z%>3E|-RZ;%UBaI@tw!UZjzruCyvw~(IIiV4yrvB-R0u73GIr+1fJ zUZP!vmRgu_ESkZ=1NUk1P2n`~ZLe9*XFCU{dvpkR;)8=PfC{!f?cthsc5Ot3-&V*W z3h=Q0E}!l6cNRof(!>0jPwjZi9X9jKTmG)%SQ&(41ZZ8Q4ys{ef&IMl8PFkrx>o=z zuzknFal$c4G>(-74Z?_x%T?NJIUJ2Mb`==#$`(Hw4i@bb6NQrPRCxCAzwEmvtnqdb zn9w|vT8d^l@Eeti-6hxK6-BaSPm3qmw(sHCj{RIXLb*~PFi@ZA6uJrih0T`3N%oCG zgK>P@|EQB&cw`>K-5`(Mr0kU~ zpaO=PMwETbr7APGh92z^XZMQtx4dk+9F7=&AMGI1TX53Je0sm%IV(DicR(M`nEs2R zMPSB>>!BF>B|py|of9kY?R+=er(o@7UpWm2znIORGytg)M?m@cQ<)$ zfA<&<*cLeqx7Hq>gOgmeGN;}f?4&=qI7DoaH{$JC^0!1{=n^^xn`JxT1biCkpFtD`@Uh?y=M7w#cDEa}20zsh1+Pz*b45 zNRrvV_cMDKrb5HQfu<3Zg69=a%c>8b^m4syn$%gV-26!)narH|0_PdB?%2h4akiRD zT~$dn(T;Z~l-nYQQ=D=3(I7Ecl$1y!Q7FmncVu75R3JoS(2{!fg%J%c!Gqpe7D%TG zrTv3k5-8Ysp43`;r7SO*uBlc;&PbQbApm_jllJab>FtxlmVLl^m(FhQg9XnfKj5b) z5|%~qw~u+RwmbVw7-0#MWTgNHUel&!ADri$ULl?ACwtvu(FU|A(`kn}VUZI@$tdb~ zJoVnAPned%yS1vfPY&^|7Nz~p8y2U>e(?V8zy7B5)ArA2Le5*OA(vhLux)8LI&zZU zDjaN@1{59ABkD2!oz9Ph1w4x?A_ow#eb+ zOOnEKJP0UW^a_gvF_{}4^~lbp#IK|_HJ)TUoZ%8N(pSQOa%;jdPSg&}Zy@W{QX!M+ z5Bq-y(AlN?8LzX29CiZ`gbnP_!od+HN|=r6pJ%>;iSKOr_wu&O&!P*tp(vO*xsbwC z?1K*%szhKg_|;3VKHZsZrEt}l?7229TY1Up)L?9_)Ij5^Xe5w9lzk6O#s0WkNVQ+L z@QxfZiGF|Q^+$BiPk$~92+GzAjHVh^w0K6leC2@m=XedaMIX-Kg3=m| zd{X@3;zN>2`?J?f#>cKM%uv%etC%^J@PMp1i_tG9vNAM)FbI4c&#A*tA@@44eFq?w zOtag~^q_*%5Euu_XmGIoQVXO@xdA{l$aHZ}a=vA~~F~<)Z!cvHI%;d5Hk* z?7V;QtM-e~&F0GMS}qDH^(JM`Un0yZ44JH$TAE{3fl|w9pyde^TTlx2NezI`d7%q| zM_D+w)2JL#c04=>JFso)V6~wPvcl{*!@1fEb<4iREt129f42Lt7%!o8uZ8_(cT4)>KU+X|W>>}c~i%~uYSBu^=r{u1804se^~a53VfkRz42oR}E0 z^J4~|zC!9Ns`^|PYDD2_Ga8X&#DWf58%tP50r(dA7e*K@pI2}^563H!OJ0yV=S>?A zr?mG;4+6)xu<(RIV1-K0!m7j3>^oOzfI5PoO`PY<)D^t-m$dzISg$|*v_m$xC?5Cqk z)ji!cyJuAzFK0eL{+i-ETX&fq7bN0SWgZwjeTOu*K{z>B)mkmLd$BAEPuEAR#I0?= z&U=oae|VfKW#t)bnd76x+wtXq`MM`g;I1$s;!b-t-ch&&Noh}y#Q(=~a9O^}r)VYz z^n7ibrXD_i(nc|d*|2wP-GrSC_{%wzUCRXnCCEPUh~g;GsH$rt9FtF+KF?|wuDWU> zqj}lL?d6Jby6=CrqdG@?g~eMo6?^=}DW`QZ(C@j9gTHe96vepUTpYF6cSkd89g@_OyRaq9;6v$xkexW zlcSp)*E2ec(B!uGbSr&vu&zR1TR4!xd^xjnT9Mk9kTfq+seOc>3;EYHtsVy?oB?>R zZb8DuNp^yRK|zZBpo0mrZ;Pf}HA8;2NYO-^AsXdt1A@q?DZAclYkl~MrNC>0ODi#b zI6>>opzYk^$t9ThN%ztZS{uDEM=p5u&FFF_njGe!>>MA5|y0a>D=&xW?d_ZAq#beFz^lL zb7@iCg21{!kPmNm&9F<(SZ(fU(Q=Oq_?7Yr5{8EWqTpLWabuXd-f3G6{v6~@)@C(? z&51XXVxy1iNCX7QUCxPhj-71`bNWfk$kaBb}I{ZWjiEo-~ zX9a}{63*tDb%Y8~8CCX`T%g0p)TGz0e@o(&p=9F*)hxj9ujyOs%W^`)w)H9?n3$S9b< zCTJa*25#$c$B%A5(Owy9=X_(E+03mk!!P+Z#&ymcdRAX0NFO|1nb^*N!!65*lg$d} zh<$9+`;|&y7{JI6%ZykIVgeaFyu77TOaF8#Y`<5XFnhP#E_2JvC#g{X#?CJN@WUHx21yTjA8v>215lNQ?=dxU#-X? z`SLfte{Z!v70CAzVYy{!Mi(Y zZRTpKH_2MIM+_`X8OpT?%(Y;OXLPjt-PPNibk#O5xzZX*dSCN@J-h7| z{w!A=`6Wad0&!1gHhU_|r~2KO_2K9ZX6y(ev4&m3Ayvo&uJ5$4m}WE5RrIyqn2QPX z3WPWXLB2jy+=vGbZ!@oac&Q<_7XAP>ntAT(;Pl%}TS924@p-*6Fe-BKXoap$1*9hP zdKbLOKJW$Oi9y1&M_smYfZ6V&12yG85DK5t}A zEbQpoI}-J;N#I&_a0YLs9V#csbT0bd6a< zTB$I%LV61q=}yO?boKrvcV9oG`bXGff@I%zL5(#}1e5%TA7O=I_QY}1hMh14XfVG$ zFQ|32HP@@-{iLY)K$pazpU&-5&gJ9yh99)8dtkA~)*FZ{E!v{LXpES_e^54_(%`>@ zd5W=mYU}AgykL1z*^$@Z(qILg9uD(scOMgcc&>Ixll~t=4fFkXs9{PDZg!|ZgOW`O ziuwP+(oih$50-^s4)%YrG8A+CgH@oI^B=4R#a#bj4JhXR2Wvtx&p%iTilK3iA+~g& z_^(|PDE@2L6pH`aHG^Qzzjn=`_^(|HDE@2rJrw`7``+69Ptjk~?;YHnp{0MVTSD<) z>sCkvUsiTKy6JrQ!sw@Rtg-FWi5rxIpn=DlQi8f9P}n zrQ>4l2`&9g$i>dY1=?Hgzoc9oP0XOQxc}0ETI2pp%hk%+;!oLMO0FKz?r{I*;s(Wk zxp-SRLyLI+p#ur|Z1KSy8oZi=<4^5><5)|$m|1*qr4(T2g*caqqrAnR?y*y{Nn89= z!omshA2wM#6H6CL$Z)dBI(%@IlyvZ<)Tdt+Tnwy$p;q}Ne4S~N;U;o6FX}&i4T@`77%x1Q#SE@V{PsV z83%TD$f2gIg}n|`uI1$jRRl2#CFE@F=<481$)@`sp8v0oLAws^m_8*Rq~HHH6DJ=( zB^SiwxcRs#d3bpLBz}GYN4{t+x&m!e{?|fKz00^ zcp)dh03XjETTpxdCSG1XN@y7$ME@Tif9*gqRL&34^QSIE4z2rdz5ld>)`Q3eAddP+ z_E#U2!@vChaDem|DuZ%|>izE=Du>FT>A(95#nAEjx9s2i&-nb+2bDo}L3#aWTp-Q@ zt;5OxPY$&Yo%cWWAo+jB5>oGvXZ_m{O5~3K+LQkrK$}9%{OdaZp$KWt12y$m^`Dmi zX$92@J%A3(ALEdkP_@w3Q2D<$|IMLf|82tuSzNgI{yBz{ht`M6cz8G|xuI4e=QC9H zZ{p$m$C>^;{*&|a{L#k`Isg7me|qs>lmD+1K=VKB|I9jc2LCJl+ueWj|1^ShK*)!9;xL6uej079Zu*Ntn>(+oz`SIqS2Uf|G#7=42$KNu9bnsjRbF2LjrZ0pR$3{!P)%?m7VKB05ZK=Md`@a2aUD_jr_*>Xe)Io?C~q8?Xt_ zO$j#}mQCJ3=aV2x3o+Tzz22ToMp?}sD4J|n4j7Lwg@Jqz+!{>)S|;H+Tb+SAiwO(F zbVtWvQNrmB7=m3qG66JF7v@6D{aOo@hwWu$>CQ~wt&~BJMj<4NWk6@|={<^k#g=`3 z<5ZL*Z4*esdrd&ORj~j3&Po!QcSx<4@i=pxE*UtU?VUCd$Mp+imWGR;>o*JmT(#5) z!dTx;kBj(PLqNb#xZUR_&;%%~2VW+tf?5d&((iT=QY>;{T#*c55kHQ~L86;m-_p$! zKLF2BjaDuvv1bQ4S!JZJ1oBDH^jNMF^4;DDV&F2t(bd;mHB831VPzNBMvE8Ufi`qg zCcE)5dm*Nw8)Y-%7Kwc5quxcE(R{%!l}gGlyKi6^B#_x9=QzN|sP5=Y1M5016Yt>z z$Vna2)GH%K0kpYq_Vlxi#UV|ax`RqYs!h~@Afp0Idh$hHed>;K80{+@-}>|9(IzxM zyfS2sE0iVt+nNagv4aw7c+P@Mw+T8XaNo`Q<=Bb3 z;Q%tKY^jhju5`-2j1vgJ1GuA4ja`X=$?TNCuEil&!Y6@9#$qYaiIjkVtOORMaQILj*HlxCry3je)U4xty#uAtAL2+f|I-l{4{TZO?!`4K#Y0`<8#b^?JQ8pXt|*l8Z);URsy=d;9T z5m{uZ3!Fp_{&h`I9g7Ct+8OUyCy4)`wE2VGEItP^ktBXCl!jhX?O&x&9KVNRCguT`SUhi<&6K zxG!iw896N${<^MN<_>HG!O$5zh;V+)su2z0QxPext-;Qmioq=}&2NYI(_N$0x2tI-6X>a`v2bXjeUF+S^4fT3% z9EP8f{AKbV!d9de81i=1oDmdb6=>t`AmME5=V(9EU@>n7DGF0P{n~R#MSumN=PRI> z1UvLm%Ku3T;DjHnj$^kLnr{=rM{aUh;bO_BA2<$PBy_(rdKm>T-$q3gQ6prg)SGrqQFD8ykEV!UR;iZP`?iC2!vAWULmc zI-PdUPKm^ZE)&1C)W8p3p*)N4dkX_%EvLK;8RuQl_tn^9FUwDhuZQ|4h94rM7itM{ z@X6(GH+XuQYJ@!zmcwhQVMQYaEG^!+sDFPcp-wp`w}b7Q)e-iI>a-6B{42dPalm`( z6=q@$G2@GM4iY*T|EfNJkc4>0Ymty4>QVpRquwr&ovIIa6))iVwKsbm^Wzw90%3cr zuyW2a@O@Hvzif;EX*~SG(?uqlaS^e|8_|z{1e18pt?v7vh7Ekj_Yqzz@DHwqr_geE zsMtbQl@fZ4t-!uR8a3vH!F)Zzc=gW8)W7G)^xMkzACSS){5+_Lr0R)u5O7+A0k5Z2 z8U3_;F7H2r0on%~3^jS1smAwMaKj?(g+u z7Otz27i?~;5TXfPOC+#Lr_?rCw^X8HOge3A5!()aJo8iyMKhuw|yI*Fz z2@kmK0ER<3g3>>gNil%!q-zkU9*WRt7?>3Ettis1tZ>qO+Cn*U)FOuBLH#!6 z>07qR0f7plaIa7Bk}AE`=#yDhTJ2x?E;v%op5KpZC{83eEPd0WWd*ti*mPT=iC3sf zz|z$|;*|7CQ+Eo#d+K*~PUh$~OLF)Z2cV@FHZf^5ILjFYS93uUsv#Ou|L{uPyWipkT^a>ESufYMDPH(&`Wa}~@LACLXVgdhNwG{Ld zQ^0!(iRL|~nH&TE`B;-FFWNaNEAARdnu?W=oHQc36QSV@luH(a+kStSj@O`E=U}#lz9!Kosnwd5^;Ri(Czva*SK%D72DM>PZ|+9NT`mFMO7J@Hua@ica-B z#n<|Bd`_{9B{{@+TeUDPOx}skp0`aZb+Du%{|G}c65#d-pu<N>G42Dgs!)xBEycJ>!b&uPMkZf3^m|D^EW1 zDE<(iAKAb=g_&gv!3Q{F23mD*!ajj{l8p;uv2|fU2hnGQaqNh&cjV|eeJ{9euE_hQ zaV{*BzhRdRM0Nfgpouzfk-i&{OHueR-mE$~q}tc_>q_=V!`JXMHgQiMqM1EUAJL6j ziftZl@EO+}@&Wo|W;9~h=1gr3khy!aWs)_16=?T5{6R<5UMMFT0Aume`0#b3M8(`) z+~C6)p=i3{sVwV&h9h;*yXesb9@4z7?m;8@0NBMjsVJ5bV58hTsJ=th4~`_>o97?` zXnq4W7{U%XJ{4Fr&Iq&Ct(LNYqlje?F4PjHdm~%cN2rB!g%Eu?l*!J^0LZli(KRV7 zu~5%WnW<@SaH0mG$5SB@+}lOW3_6o~~g&B1l?TA%9yQ3s+s zsRrze5h;)t(L3s+0YI4x6owj(3{Mvv8ZdY_?%R8v#9O93xYX$Uh3PqW1CHW#)YD<*(GJZ*rZWpb({ZwCgq z9uEDY<=9Il%uo*JxKszHG&4|g9+ zFvJBdrO=aKFxBR`2xwG@mk*9LW&POnKvV^X;0IWYn)x9+59AXw8LOcg6JFrByzD@^N8GqiQlj8!E4LRiX)$aB1(Au{L3tYDGLlKDxvtTaoX7kEaZR z&3VlQ%ApgI1SyMbC0HoezmO4F;~y!o3wF;oHvy^_ZH9+%8T4g+=0?G)+$@S~JEPTt zZOx@18YCoR87=hUvpE0x*1(7zOj@vx&>swQ9^g!5W4hE&1&EreFSaN9+|ADLRb8|= zscLKJu6Qd<@1*rZE3gGJlM$h!IXB#^c1>Th3Y%7ZdyNroH6%1~wdp-Pf((=|o~t!Y zX2QGL^ZCOP!_T&ew0L7W&#Xq@m;f?C*^+Qe^h)#&h6E|&Ri1?7LgJx(WVBQ`ZgVDl zXho%!eU|iFsIa*9J8CN;pAs3674vDjd;(G10n0#!<6uN1;qbyL^YW+(A(bh``vA4-8iJ)l4ewx7GABdPYYc22m zW20un=)j9Yg01rnjfhhZCdH&95?6el4J`f&kU$G%hkMvX*1x3u}8%5{SuVVSc`fuOEdavICUSKQ~T}} z)kkVlFUboLz1qeaAj#049i}1=6AbPqpOb?VnwYUNXtDU!m=AyVo(Tfh`10lMOFx7_ z8LOUf4x3L|3U4dMN8=sv@ASc?N<+VstZPEJqKXxi7ccv|;Hif(jW6rp-F@1a;MvqB zNa(k$Gx4FL+|C zj5PaSTa3>#C>3osjg|_OciXjp6ja;zarCt_T8{^KfUw72UY&=(ZdN}D`8Nn-3=#`* zw3t94*4LUa(ild!hsk*V0dT~qRql}EBD~L;Rhr{sy8C1c>%@_9Mr>(QfJDqRerT?& z)|eIcDEcJmyi_GwX~+TF(jvP*+92}|b=jY;qZ@b$f5cmo{;)$BeO_+zwoTcq zpD|RELE1_TFFYDy8)sCkyQ-k%499DbP)tLV{jMDZ^H?o4B;~nJii|`R|I_8yM%*US z5$~efVb9KQA9vs__Nd%=k*`s1ii1FU>?wUtbx&_~jWUU3owsh8HU>ySjR$;$h(Zz{ zHIlv8gjB)OC70XVG^>qbp1BnCzBro%Qxgl3XU& zt$SyC_pBpO?>j@zed8hIzR0&PA;?O69<;Um6f1*yRe>#=a)BQ=W>JrVJ4c8qNE(R46-5ystMb0$+_&U z!gu7Q_gKd8ouoWGue-iX*J<;$xvq}o`WjjuM$D|E+JT#_)X#T?&}=OQI~|4_cZwh5 zk9}=K7Rim_wIY>771d1DHu;-uxO&_ovX`{RVHAmZqkBx;(voGSi z!d3l4dR~`z-|;aqBnkQao_vVB4TIF$zH>>UXX1uCOoJvs&aB;W>4$uUqA3}hy&EVykFB<+I@r-O*@sY19_r)Xio*O7QCI<7v;gM{W?F_yrh9S9VT?rLIQt7+JnVR>U zWUHv8a5>bg*@!+U6wD;HZJsnNcgVD%{Crf1az?OIH8sE=*@6T$T(2EoRmpA~MW$~f z3Wp3)xOgqfp3dK$$=Zl$aFTT?e-U2U)eRh|e79YA2v{P{Of4vJLdrH7!eXNhic}wu&wT{~tnR{j*bQo0054^Xg zHx`W!r_qY)cJcahEi(ms8e_?6Gcu+LY(HuN^CWmV9ZtvNX7QDxgm z2l=8+zDbt@w(b{pWGM)PycBk7Td>0gJ`-jZzAxnE=di_}LV5;6_I{KL@+)%&(|=AJ z1v-o%6lb{WN2Ob>3pCfCd2#N&u`jnY7S0D#=h`^6MjWBjFG9xhEvY9y)=4KwrvR;0 zgDz*QXs4-saP;n15>8q)BV_R{1cz{*ZIQSe0k}3>J?QdYwf`vIArIeFK5jq+wqO;* zka$Dx*--2rLbIN}&-4{TZs;hjIauL%GEt;0#N{l20ktP1S{*aSw$js-1j&2;V`exA zqWFs!Q#-vubwcP`;|9o1zr@@njLlHprMTusqpQoQN-V&H$I8Dgi_I^Ye-ZA#9X4I= z@7~yq=*5C;GE%r5msTg55|ciiP4y+Ab8g+%h8;L@R+&tf)#m(@J|P2Nm6y9;(P|7O+R5b{{Hc62u*0Q zv-~F#*i@T4ZC+x-CD-=H1G+#x#o`@yX_uO?4B6zaO8-daT+PLp<5ij2h5zW$IY|#B zI(V}Ge%)=Edh6y*Y@$II$XAZ1ItXAx=emY!bt_IX1TR<3NhE?wT5Ms9dTxT5q5LWd zGby5%PLU{C7}r?1tD7WBv%iF6^qfzaNSuE5JQD=Z8$xH-J*4g0Kv}H$_5AK5wTd<^ z@?uEf>Fl1dgmF_b%lC_;owe*!bl2V5l-QcStFjC?Nz4N(At^Pj;k+Dn76w09c-+@Y z$uxXa&(vr>x6~GbemLCSXkTIWvhvx&3cnv}_tS`J#WIPo7qQ4U#%1Gcb`Emn6!V#{ z=?-Jnp4Q9`!Lc)1m3#scJEed1`z)k!(yytg@PbDNtWJ?Kl#F=RKu-__xp}}ztDfIJ zM?#T8Az5-gS~@WZS>Wor0aGV`IlkWZZw%+7pgAKhw=U|hW%uo7K+hSGd-;NB z!?1+f0L)llS89l}AGH~R88_k&E96_sw!7@aNmfO)>%)N#kaHV#UW)tt=5&ki+gD;P zQ@A|(aMX`cU$?DgrJlT6vbMc3065^ERNp!D@vZ35T(uwm^LOgn4I? zGgNGVG4s{bm+m7Ek#2RmdR%3KXE8G%4T(prsgsS>cL;4y*yI!%$Hu6b1*(EO($!g7 zVRiljsxh6g?%5U8ye{JD@6bYpDd;%6b)-i+7_X&m#^WCd!`KK?kG$%7+m(hqLa5R6 z-jUd7$FiF$N2QoQ3h@IFd~owz?Q#RKe3my0AEN0GY5RFTrg*0T-S3jurIhBx$@iU0 z5`1K-etxR^p1TQDF}Yw_?iS_l>Q4CR)E>HES+v-kDSyjC`=C z`fx}njH!wdbJo*z{`1H9>5Z+num?A(L`DDL?2Iu^K6Z(~obrpgw3qwudiOD%Q!QKf zJ=B6;0Pg3;jj7&tHd8^CJ#QzncchCvV3y6H6ham`XL>GnGkAEyUSx>{&@M3cP!I_Y ztZ7Dj#7ug;bGa3Qo>HLEDg6~LLsTT1t~9@Q9EI&vO-~5NxdtVP-#>afct|{eXo6Rb z?{jYMzUQ~**uGSROaajd=Q`^p4+*rTv=wGh-<-W`PYQLXrQesKNC9V;%X`v?j`}Q| z_31_OcuRKHU{h=|^@NY+b718{ep4Qup{=Jxsrx*KD+|%$nE2UaVaxdib3sFb32u0V zxIrIoiIJbj`jz=KBS_wJcWZ1Y^ZFT{0>`^zE_Iw0eA#zq`f7E6@!D97CAb$1c8Ub~ zj&ddu+N8Tfw>TG@90jSmFb2S-D3@mAui=nK%~i;V5`rnZ0XZLWgwR#;z>z3dz|4Un zjCc5m2(7OX#uH*?^?%jdrz51XyaE9Wi9dgqV#ZF$EV)hhJnhuyPWKIWSZ@m7liSx? zs;~AxFZBGSJnHK68sE8v9C%6u`n4@dGvd!^zq4MiddlpHpEWYqGGWnm4T&gQ{Z(pS z*aWroz5#Nv6cJ~Zwj#mx&uQ-&aIIOl!0bRuV*T8G^6KlNfTp9@hmuI-2KQM9ILPJV>RrMLb)-;6$|~uu zXh8te!XaMxIapZ$hvPt6VXiCsa#adYb(d@hxw?2*QrVpbJZo?Zmk3G%||dtI@8b(m)#wQ@dnRiQ18r79MWE|N^h z&yZ`$?ct4f zz&XtVjO1C7LEfbRj;S_ByfF9^H?xN1N4Ptrm(E+_h%yQcY&*QBJUJ?$W@bFq4WOep zaJAMKNfy(`k@BntX0^`%OvUze3!xu5H_pS-_Eb`7RipI%XSS^{6`y({I8CZWkAp?; z9eJErBM~Dy1D_zbUCF|&uDdw_FaClw6T|F}8_rIhwTXFbsd9F~AOL$GL5{F_z^QT{ zj(98fyg`YMfLzgL=1UMZ(&S}`efn}P)978)!UT2_n#lt?K6vp7{?efM6YCGmJq66! z`X9XO)4#3cQ&^L|hlSqYKi57?dn=HoL@4BLGuuyb4V&5hL5J<(?b*pt?#=3KIiJUpGb_)t>T)%`%v*DF<#>&VEyf z<{gitUxg(k>mCZ?PUzgtBJvO;X!>}fsR)D>X>F4&ay;mz$x2dk6kVxM`5=~z{ zqHT%MhMyU+$Mi8nk*eQgUmHKCy#+<&gc0KcuSYXUZR#n_`F3D^esl{iDEUEaN;-&bM z`k;Fs`0Y*d)5MLC5{ZDuyWOMcz%c>;Ut_L5u%wP84H^QiADh602u5ZfXETsjGg(;E z__!yPlTq^(vIP<57|<`O7>JEyBP6CBk`|ixCo|Y3Niar!gO>(>znAzF+8TF=-lfX- zyAVe_P-$qPCvX4bewR_14&%@1SVfuQ|^`pdo+T*yX#s)bI768a=-O zoliz1qOBX2xK9FNIyy|}CzHj+1?MnvojiSdYsE62IrpuDrBVEr5ycC`JJs(X-+)@< z1A;nJiaiPFD0R%}qMt&3TwBAbh5I*f*Bq>zt|sKZSPZ~Z3zyrZE)VcWX8*!ZB^NFD zg+C$e@WT=;!KXQ0p_s^GtEw%?_O7&(wYES%O?-2o9gXzZs)%x&x^8pZF9Z&C(ylyG zkCS`UzWt&_!f?v>lkY9_LT`UWVNwg{z#p}T&_mu!l|*CQ9%@zquxMLcR~III2y=a5 zJIkV7;FhrLSNA-8iL!I&m|yb%-O?kfFjzT=c{tnRK%h_{U{(b-6N<>L zWJCmr=48=?u#T?o!HeXTN;b-?djnMyJe89-Z>m+

0FWQ8kIGU661g=&(Y`aj{x6xZ`GatBnw++YEJ43w)cB| z$fCZ_ZB{}BpReoPKArC58Z$!*gN#+ng&lP;FlptgU^~KG&>Y*6y?bf~Y zV5*YmDd{Eu^o1|`r6*C8+W@bqvN1s^&lPc@!4%}N?f1F1Z}u`)NmLY%LHN2o9#qe! z(fH!=cd{qWhsT)^_W)kPSn|3BE03l;gf$k81c_;)tZ^ZA^>D%l)$>??mGb`@#&XSW zjsuzx%qH&=F)_vf^Q&4X{fN#jR1oucVD^Py?Vg~MAn_@==GY;*{e_<%JGmZVfX1{3 zs$_N}pkGRZBaB)6eQpssR_Io=<^DrC$7F#6_>lMs!=iyZqA107=B6{@2MC0OZ1v+k<_BbM??%5~FzN75-!i-eTLEI+YMh?#rI}@>3>d(^7cbJ4K4Q z`kJXl{X>2_NvP)SC)eNm;17e-td}UBGDbU?19lthUZyILs9$@2n`p_`7{a| zhT)N_BMB6_ou-FKoi7YC_M2y_R&qp~m?s@NQ5kmCUn@#5zG97z#@nvk?Nl089==|; zz4aIx$lT4)TyddjSAftPcu>-OVPO#1NCA%KxTnHn7-?E#p2R=z_-Zh*JM0j;bpc`# z=!*@Tr_634VHzJ($~O z&I!bin{T*%ffRNek1!`B>9NJ!6-#si=*N&ak}Un7@V{~hbcR>u1d=A3=?8?z%it0a zvm-8;ECh%e_ibWl#Z!oovj{Vq{A}=et%U1_hDODJKJ(JbVcN{7*G&hr%saR}PEa)U z7C!5JN2|JS6k|>_3PYCKQyw=!z)26FTg#*wU~iPwYA-QcjFOz83*+Y(qPG?7dTTG4 zFKdNNmJ( zBNbsLhEKpM72dT7pKVIG>Ym?N5rFC}7g)|+KCYAj6|$d%1;=VY=3*|Q{8afb%bDW6 zj01#oR0D)=Wap7c+_)Iid@m60;q2E%f;9q0Ug^8IzXZEFPFHl@S9m$Nw$D(zh1kPW z3LiWid@pY6+ABu8th!_KBRC_9mJ6GPU0?QtC9WIDXWTt`pEFsJ4CW6ED%>7Zn^TW1 zDN*DlCMj3q}+K-t>@P3G8bUi+8)W>#cpNS2%tM8Fxu5VZfGQNB~LFR)OfE z8JGeBXNo!1)i0e+q$!eH3Ps2{z2Iph1k*x7xDa19M-Dmm(LK3cbnbbEQQK`D*(2J$ z0-1auBWg- zkgv39ORNZA2qT4)k1M9XLlJrXFH5<> zUC-2^vjZdj)w%}bGjdE;e7_o>EJs^Q9V5RFS)oogos0Y+V#=iq$N0iLGI4n`FSP4B zeLn7qgo+atAR7>_YOFu79JZuVkP36WGojuI!V3rEuif9g2_RJ`W6dz$wVV0|%n6zg zMDhL*{-ry8{9VSQnvN1`d#r8y)xdOs1O?KM!9CQFpE)!%uID|f5Lp2A2zneH9loUAstQ_JwM5AR9%9>WN#*WN76t!-)2s=spP0MVfS9CryO zVI28}noqp}Jaah-HsY3L`|8ny!rnI(vLD;()i{y)c>8_RrfRce`W$T)1F5^|RRGiX zEjJ@4JN<&~gQdBkokF;2AvNcqdXe*NgB_0897kz@%`p*cU7vOeph7oer{< zM%@&@R4#ltW}DBHK_vM+%_OHLZacVZ$i=p34Nu#ac}J!F=Tf6KWNJPqG25@g9tqwJ z74x0IW<-atn!2aU&ia(hS~fyTi!C(*OO1kpqVQ5kBzgPx zOoq0_1MD)^^Kr9)(APvSX%HCHxFIGpXiH;F8elzh!@Uz~8)9k?~ zm;vhHu6$fcq(HvqFDF&_XAcIOog+6;{V)UsSnk;w?sdL(#q@IR^}r~A|A|s72G%&L zb=Lg)Cz+~6l+v?IvvJ7~e8zq~VFM2kEBUqaO8%->zLhiseYBCJ2wMDZix=Acua8|- zyS{TA0G8!m@X?(;u0rotxZk2R*nYk5BF}JbQ-T^a;yQli>?v8Prl>FS3P-a_V``)@ z)FvV8HzWX;^$lE7=2I=k;;U>d_3HU@4WMLGlHy&En4Kg2n_ zkAmzhG7UKQKg2-*e^lzjwsd2dK!7FC^}1-YYBq^5rGT8`P0`B)N4a0xS4F!-0PhW0M z44l}-YmZvis`Q;~GZ>|dp{T&^P=#Q$0oYHG! zKd4xHcAz*0`Er}uCw0~NW_DSD5XcMR)r0Oz7TfUbC^c6rKtqN1ogkA13tq2R{a!s_M({j)Ytx_lEAC)lX`Iz%Zkedjv7jnyJzs zw}q6|o+QsHY&(X$Vx?Sie~?J>Rh+-2oxM*@X%jf@j3x7&z|quxk+QW^F_m<_1@{Pw zikr-l($C8Mbb@@Nr?fyd-mpG8oyPfIJ#eNg6+Ue{3SQ9MYiAAb)6}4kgXvT7Br8?+ zkKXga0VCIGQr;goJO+%H;+{ipN6l09Hnf*vm+pRk+Jb1d-h36lxd`P&{>{!9HM@ov zmG&r$`(|5~ZrG`5KMo_omR{2w53o9j#G>nVd{U@LmA>t5h{}q5Yh>SRs3%P)OME}K znLN0il@jP)b#{suo|MWTapEuPRKGR#q2GLGampxVc;_tlvt8PL(Bem{$6$k}2jo=n zKf<|a_Te%krJx9bIWF=~YCmqLZ_~5Lgz zYp%lI7$(|7dk8}r)x||K?t-r(QB{`B&|}#Dn@BUjn_Mx)&0!^4ogRfWL2UXKaqkZN z(Xzbku-`2$A)(aN1z+bd zywcQNJmmR)8~!;~tjA#U$v3_`!_aVxz*H}KwM+7ocqgdra-&(=V}kI-D*fO;&*?*Fl%G{bjm5 znMjBkLXdZ#p(rgh5|A(hIz==ID^9E;$#14YS;4FrqGKIyDoe{$nmnlwq?p=`yPk+{ zE4`I<30*B>f;8aA6#qWu3DpD#$mTfNdm0H2O7)H9$!OS({h8VaWN*wm5oLxVprPJZ zR+h3eyJxqLzN_fJM975Pe^%4+Wjm!r`!h(KHgM*6=f8H?jjeu^UAw$R`YHVym3oe+ zTtr?CH^n@JoHM|$iDlDlBEnlNm1n@HCy{hu;%3ZC-uROE2u7%3#LvBt6yNZ!kM?4I|}hlhzp>=eHC zY;rv^mJ;xvr8w9nrO!L+blq{7KDk;dTTyV(vp*7_b?7#8zZ=!)7Su?bqoiM;c)$r3 zlpxt@m`+>ao0Bph{BZF2Qb^@<(2Lp4W!th9d@Rhij{ToAHQl}KCKs&hTU#V&9Ytcy zq5d7|ia6dXI7^GhvAi+ok{`L6-W4UE;0YPqasxoHZFQdrXYNH7JZT(|7h22{CWzpK zt>o^qXKFT)$yZCJ-t&hPJW;%pqg-NEN^)6u%x=kCsaGoC2B*AGjy%*CazVMp9+(mi z@-ymem$Z6aklDHNZb)Qz*|0IjNd~kyc69!@Axjhaax>Z%*I@E_QJPmu1{j6rz~wl$Xx@wo>YVDL4}| zJN~&W1wriU7X7Ro*782%4HFw#rqj2(v2bS7wV&tAq@C?O+>yCMh=d&~1+uG&aqqCC z+}7>bzo+|8m4`R{noU$O{FK%}{SyOTUhU2|nsuK1{hEQY9mZ7aRm(yO_;JYh%|qmO zR{e!xDLKV_hG~AtieC{cXZ)?m)DV2=k?msXpYT*8rOj3Y6RATJ!||@T@0OIKl=nTS zEUH2=DMD?Ksm1i<+g$N?)o-jvVVgRb9api*_(~TRHIg2j%ej^?yH{O=jjOho?SH#< z{qmaN`zC?()jabF|JFI#Xw-X!X{o4t3(1T>rla1{WQ}Wnfa|j<&Q06T&92DsDl^32 zw-Z%WRGC(1A_%@*XC077NCy^!6A!8cR(f!s-+tS@q9HE2WZ%dYZ8;k=k}IdNi6&;W z=~8_L0GZReoLP|`S=x{*sYPxPv&=hTFy9Vg>s8n$J$PN^iss#t$l6*&_0PLnqtcH^F6;+NS@>>o@<|t7`V9 zD$mZ>5l=2>19P;^CRK7vPa5WMm%_*YimS+(2zi z^H*nBkcGe9GIu+L`5STc;CqwOLS%W%ggM^(G`u`BD4=fOZLrOkhFk^ zDB2?I?uIw3lUKtVn^DpQ?8eaBHuQ`BgrFDJ{d)N;%{eb$%0(lLrk$6>EYMg5csUsu z42BAmlA#h8;Vs%jUL{3Uv4;o|l|62l8wd3H_ztyhWL}f_SfMjo>P?7z$ju`ykvgl>stIc9vKORjgjjoJZkP2iKl?z zQK~ypN;q;}W5Yu|qQvk5GtAX5rH7es^bfJ0x9{1zdl6R~Yjp6kQM>C{sxvy9>&44M z-}a#~qT^DEW*=pCQDu;33g9Y7MWmG7;U_g?;33Kr8883@A((~ns9!*3y z^#@PaP}{ZQ>kp=2cqTomSM}wLe-2pX7ee&h!9t~onivrMN*}QMebcAXu{35dcp4E! z=}HgTO-DY@px~Y1G=E+>c>(H~uMdgQ#AF7-%C9P5F3-8KYjk+bZHXry-cX01JY?UR z-Y>($@lH$md(ifGl5)U(J_Z#%S$k_tWhrq>k|I^4%NcAglR~+Hi_;Ia0`cC6=|QSR zUHkZ-KVU9zwbLUVy8EVU0A+2E(2yIur)^K%7;T6jV;@dQE0=+j*H}5tx}reh9b;J2 z{E<9$^L*Mn{K$N!VCKo4&&+3G+X=jDQiNzv;(|0Xo!v{lV^<$Z$YWz`SVY;np3sH8 zuVYSGWQ+|a+VzF3qY{(!OipLs`gt~f8cg%y=4Ddimx*#2p$YffR*Ym|1<1)g5I;G~ zF9yz0f95$gJ;vVr9UY!$h6w7k*fJz2joA!NE#opBi&KDoHTy0VE{+VwQ}^n3j9*Hh z&8L8GCZbi03^qQY^`5P`cXCfCkPlNZGqC|UL?#QLKjzwmo(3U8b)Cykc2v5WTu^tE z(YjlzwwhlBlfnj62X}LRlpxea@ZOJCWJN4_Be?Pv3pU(DK<*aNY^)C+#Y_v)I;H5b6A)2Tw@C#zBduY11=8r?G_MwIP zjw_^FSKZ62H_AAYvqcQh4n8*~(lZ+=rdq4=CxV$0mXMR5-SxCoyTnOUC4b&s1qXWF(8jray%DtGDGUg+` zO4}V0;uvI=oh4s1{Y@e+MPzSIC^b|*QZ=M6VY*m_nH&tfB}63EkNHG z7Wb!WH+(1KZ5H;%gdST@dpIMTdi>}43Typ`txn{XnFba8Mc;f(Td7bTs$43VsXq2iy8SLx{fE&Ci1K>9R^QLxZ+(Ou<6fb?A0Rj2VQ1=G7@)sj|R zZA_gf1xz&1(#0$3WAEK-Mv>}c!FJ|BIOOvI5c_=#;*atEcM62rUyzkHydhWF2t_<3 zmy!x6rjt@Xcy}7fH0XE+(B^M%U^i&k+$H8I|1>o~xh_H%*Hw8HYw8npUP|kE9LBM# zB7wWo>lEE+?W#jZj>hs39A-yMt7O>h2*Yo%i`ksGp*{nmlvA%n-E+>+{P5+^v0_EyNVhWD<613NN|rR&vG z)#lsR_mp2Ya}S!|`Iq}mEtY$^@?()xN0X;+VmkPRj+r;7SIWHyhuTjzrH}uNhCQ_5 zg2>eg^b0`kVebA8cG!LnYiTE5(!6t3SqcY3M4rwmzGar^H7`4xpJBH=EFu5&$c5^? z$^HDUm$1XK3W5{4>5Q5qB_IlT$#q;}&3K*oUQ8wsRzE-L(6(I$M6Q14zVr1-K^(Bw z4Q!!tzAN)G@Gs02tbD^NIwt%kFLOqi+9UP3gZX&>TLP_>wszn3*=^zL>Lal_)fO&L zdRoODSXmsD_DH@kJWb~u^Y9B6?okjE0g3o~y)6WNx8B0#usriTy+AP>$I@J3;aw{^ z?m=6^_1pK?lLyM;ah=(=P3HhL%=F-PRFRyEkeZG@+*a{-HY646eB-n81znYXs&wXS zzBJpS<~VWEz00ZSk`UVugVdXJ2a8XomD2~%_nWXja$_}G?rJs3F?o?Z0Pn-B-#?Az zsMfu^s^*GaHFfVy_cgQ|N3&$aO7#G>ZEFAPM!UuayIyNEvCsB&@JDDa#l=|j90L-K-?#LJ zk%Mq7U{FCn@@NA7D#+6`-}A%{Y9egLAH(?t*U68_=$@%(lmDDk>G8V7RxCW)rsO!n zeV#GXh~jdf$kb>ZY(FLnQ*&K&UwybbfJ7)PbcIDN{0#wjKC%#B8f^#c>kBsO<((zL zo}Vc8ch_4*RNG#i_Lx|&y4p0`jejs z>b{ulBYpj$K8kbtwB5P4Na!`rQoelloUCl#(cPuuFsl3lDs47WaXB%?Zs(lz#BX5{ z^F#ra`};gQeNc4I#O>&uF4lKbK(GyYBCG%yp}tDVp3QKsld7))zuxm$OWHzjBWPUZ_|3mnVWVB4AAh^{Hbh0xTqndE@4?P z&F|mX$T=;`vH3L(A=T?KhxD!FfE2 zx#&b(#L*JeEiU4!PEJs) z?Q`@27w(b>paZqP0S3wgzT-Lfq)epzpVUHBOOoqR8F*t%i7lKD@6NhM-W0VjtTqI4 zZYA|!P4Ey3Nj7Q*16HOlT2_0)GUF?zH#^qx4mphN7Cl>4VBelxg?oVxoKbf~NEG<- zeGAt+q91fq9`W3S#XPfhu@T@t^7W-Jo$cs?7W|U^B?4k%BJH57ji1_U^M2HnHxc6J z9HDAo@cmY1h<}j zZu74h(T!;=EK#Ki^nqU7PSJJzrr%AQMeM(t?Y!ygTb&byBCVaFxNogF=NDY~Pa$n! zGQcG^R@}A9eeXu)skR-M$J@s5Jvg^-aK1Ex{BR`)ycez?OEQ;i=eGQ-Dz@4JkzqeN=b$B}#>2BmT2v_PobP~Y4Ys-SS^AH#oO zT3{}^*JS5q5=ZX<_qteS(whBz8O-hUDYscuavA7iGdh-MiMfJqY;5~+U|?xUb~I3R z7YTz$^|oWQVx`&(&v5~4?9!6>)-$7F4Jdf%Ks)Dii+$4jmf$@x%wYXE$0k<2yhC?52V>1Ey&}?;C?s3C{w1S&U7#)T!s0 zbd0QPkAIU!i=A-L*H({V(zHZ+K7HPjbhD!*3WXir_P&a_YM1`xbFbNAgO?cN$F8`8 zkI~ZMe$`@0D>muHhMRE)qBzKAN6)pFNy zdkxbtXUD2K*I#s>8g@EVed82oAP>{5`|4Gc{ca2iYxhoi+*a-W^S@yV^qqZ3psJH)JK0S2c>Tjf-3Qv0r{{T@MKBQB=XgfC9_&2R zxiw0|*c9eTweB{>z0Zl24aPiNEngSp`sTSj+0N^+>FTzJAle}V>72gICBR~07HV{9 znVlG96$N?Q90|OtLTaFmU#+a#(;Dtb|Kwc0X`ID7n63T%H0fz?Skm+1#UMd#Y^)jW z@L1r=X}!0Vz_ah(^2;a+=gcB!Y77+i}kfc^^^5B;uuad(lSzY zW+p7=(51~QjJey5*5Jz5pG9sNR)R`PWpCT?SJFs6JMqlW$u{0ZS{j>Y?oy4LvBqk1 zOEgVcFWcxiBk@ekX5uB|Iu)4*oWp9?NH-47~A|L3MKAEUYQ_|RGj~kcQV#=#8k(E7GXkf4#hh# zp^B~MhcLl@R4|%uY<#mL`S@<`i=J1fbmKqKKU}j&edmt&b~9wRv9>u!W$9oEr)rpB zsB!e12<27(cIG~r&v5UUg|9kaVDY^y}mw4yH;Ip-u;Dh zys@U{Ja45|cx=FU3&ChR^#_uN;E87Trm5N(k)IL;W>Q@HAq)QR`UH8!Yu4#U54e11qBqq92)0%-55~SyQQQ!RWb`IY6i(Jxi3M`<0MGPMN*wPqRO}*ri zz#^(46Fq&rj$R3(wYG zjNz2}+Hm{RAyH!|*PwNDH`VT0soL_-kBvWC?w(40WU<&~SHJop?q8I&z((LtHs7T1 zVAHzK%xvknss#BkGn?XfhxJ7z%+c{QtGG}XAULqf;2fSQI-xHtTn9<}-9FEpsQLB~ z9N^of0t^xv!G{c{hId$78AniWU*t;Y2A^5?RRIXD@$Oa?VnG09P-Zqwc+MD&ZAI3X z!&`4!OX5u{7nl7D8VjR0#kjGW_Hfu6cFh-~WM1l;BrrO{vbuA#_1~@F{x^S0MKy#~ zQm~_b@3=F)y+5Ota*Gi-`Tpf_k@E|cAyMF3$54&fxR^;rkV+>7vSIeivoH%f&mSDc z!5pw7ZG;Eu_lQ7!-D2jQhulL7afu2=OOxRNtlHnyGCUVi0w)QF2e)@0@RUB7jngI8 zaiCxRLR?gf#omLAe|q)=n0 z%t0fH1h~<}F{Zj#$>4NjRfsmF;>)C`wMQt7iE3T=eRAvH%dD%)M@&n{YjLv<0K|I;y z`^+zor}`ACi%$3JzOeyoWicW+;*FZ~7pbPyx4i|CS4`0#tycLKen|y%UKp4LzdTDN z`cTi9PQD2UF^u%bSndrt|K%$7sC&=6bUhU#-1Bymv=SP9fEkFDHWFfN_qJ@VDGJ4Z zkHLHr>UKHTdMd~a`&3l{(=F1KwD3z13LfjgE!8Zih$?*4Iza72uY#S9{CkRt5gz#H zaU@9RT$XAU^>-g0m08CyrlqNASzKd*HUnL(3+8m14@o~mYc7nV)fCVnaoo!D&n_+I zCsZaHRy4Ntex+Kn86(_1_XHEsWhv5AEhnb=k%q~#=k0~+0XH-iO5uXVy1k=qD{P8f zmfGWvm_&KgVaQV>=J|Ke@<>1S!@T^Dvz?aT64#)!dh%P}=)p#7OJGMFx=^Z$=&Y|n zFEh;1#rtv5{*#GFaiO&l!L6X!ktnor$$V&P)M40r&}d!D_Xy-`Ea~k~ty?k!s9W)* z-B-k6jaPh-nQu!u6KZ}BnW$R}C=2K%o)A`Os_`G=y)4~mu!&9XtD+g{@WSkN5TLicY)btf?}9y3AyYu6I10?g5Mn1htg;iFJ*mb?nDNpBlnd^rwy z@mXqSoX%wVG+pJ&nzh2Q8p*a|X;&S&TDcvF8$dsyIL6E=K*Am5c9Nf- zc&4CViMU7cR}7m2Cn$z=R>=G$PGqt_MOA5LLYrZTDwF>oC;1)e1*BmgePT_3RbrwX zfcejz;~TzijWOB+nbkWoO6MeOcbX4LzlL$GFBk9r-0$C36TJqmOEp+{23$aYO0AAD zPI&cI#OaBQ-;-^ZVGIX8b-4TIIcA0Jm$LM|sEPHStMiS&GjmJTRv2t;Wugo5f^`xV^CzTFetG;?b9^dDF#~E(SADt1fNF%JF2TX6nU6ocm~C= zIxLSLSHGQDm1(v`ZzaYX;Y0G-;5>csgm@ih$DCL8LW3QRhS)0{(>4MLX4v?}a*O_l zKF)v)?Lv#Nzy;jhJ`Vo3wyVq&52Re?rZ{W&D}`y=-@{w-n>0!?6T#IYhk7E=&&53p z+LTg(2$*9O5eF(O3Rs!9#+ zU(+QD6N&2donq)_QWdgY)l2g@0u>Eyb)yrgM@JzQ!TfD?`STLNQU;Q{64lhj_43c& z{^k0wqMqX`@|a=Ibq?`dMS^=T8~}1=Dbd%DZ?q9~GIM1}rJ+3o)a^xw7XDO3-#Ou# zU<1M+d&DX=fb4zk3YvE!#p5w<&q@TZ>DNGd`S-rA)CbrS*|R9TfyJQr8ACo;w!L5b zY83cTs{q<8CRlu4(%p-Rc^GQ%EWBHBb#m~k{QC!M1m4{Q6wks;IU~qeg6-y!!(8C! zqwG|#rZm$!N4=!qd#v8Kmub))IqZaM66HOH;5)YP@LX{4)8SMGic#idKYF=>K%#dq zdFm#SW5q*W3`)L7i)giJax?YcpB+wAY@!6{FwV1C%5a7`!jMWKzApz{Yg!0!TcTLw zSA`;0nb<;D*C^2xC4M|j_k~zSk!+!S&&y+@Gb=L;tt<$p%|w;=rs(PYASFJL_e6Ob zB^~XR08|7PoP=CM>2`U36!#-023-!Xn+j*|!Ql0~!H{=7gf-X}X_rG@%}QOHVL7oS zLtaCyc-br3$%$v(8e1b%P5#uBlufc>{}r110_4wlJ^KG6;=cV4rTYJ%rLy+!ZW`9E za?Xw}2!bmDUn+utmpVH*yK4UrIu*e#^|ZINR&=%S{*Pn&A9_#2!qw4@0R;NL_aWe{ z|MSbk(aDYBf5Gn)68PWn`-J#}{$m4Txw*SqTR38QWgkDOeP_0B-Nk;s+-Co!leS^4 zLnFl_e5UoaJsnBW_I^-Y13RK<t)|8Yu_rKT;MbnNZudq>SgX_ z?t3QKI7u2Fv${X5-SwAzR*%D>{AcF1j=c5nD$;7m#mx&4l^g^4?;3yx9qxTbo2L?v zBm9oqisfG7E_AW{&5gXWdD5t5WKCN%*!?L{WBle!nLgHWFEgB~lP`?K1V z@dB$BUh8+m)wi5XgQG)U*&9{8$i?LHZ}92|lx9R>;Bav)^-9UY`X@-M^2DYS<%vk6 z_z;^FXaXM4jgGq)+kgG1*(kg%O2n*bi{80@!gPn;wz(W$-^(dg|9;bsb`P2Te+pTf zsShH4C@oDx8pQUJRIzymhkr$aw)s)>=1X*6fKa~xV^9&Su?>@MqbO|WLDyCNAJ{?y z|GghwV`blbL9_XcmvHW>S-ql^_Xk_V(<>nm(u$+GuQ0BUdhXm-?t?I#QzW0^h1f4F z`u@DVVK$WAP(*cdq^l*@?k`dkW|1ZT_gc8D-9ZTe z5k}TSbJoadl_n|?X}J*D%6V*S(1eSJG4!RQv#FwhZI5yofJeO^^cjujt^Kp#Scj&C z&kJ^Tbv~x5FZ2*3&uBXD@R+>%T0kAkZTWa^gM+%77iHg-QdR6#(gN@aHGE#D0NW8?_v zCd8QTd~2~GOXU`HHG-&O6%HzE;A=;6fyQyHwgsks`wCmh!Y&-apa;v5JDMW^>QG{| zNrZ}p`{oqMXnDu~JW(lBBaqsl@%(wCU#0rO8V*oi3{puoHjZwot5?GWaWKt}Agrv= zJk-yB5$W^xqk}jP6i6Vmqi5C3SjZniD$CuzQ5MJ@;7)l!MAsxk zWnZS*#(N9TW{)};n5|;5;o+pyT12S!%(PUGBJoakNX)FT@61_bK%r87%yy5g_^;rf^k`GcV+=;1u*4H9^Tpz7J8LS0{e?<;VWKHZ9gW~gibq&OgonVg)Ml?M< zj=-DB(H;2~DSOnE51C>0(ds`B!=kH?>8FIRFTlxXu!VAZ9HhB|6h@=!I4fzM5S`4$ zl5+vQTH-Dl_8Q9HdS*5F479W)?YHDmUi(-2kBL`Mzn>EB{Ib4-bM@rBu>3JGjLj1a z&ssntqh5Zc;5pe^BdI+F(bT;j&l%h-6Dmd;*#_gy3d|~b_o*qN)-bzZ;0p!Sd~B+z z?2QJ|YZOSD0V0>t1__|CAjYnbxBp61 zEHPsVo?*hXL4Lz&P2>egDf#XSxcQOTk2JquP|$uAkzj&XGj=RP>-n`nusd;GGCUBKq%iP{NQf5sQnyEiupi2m^tukE zm_}JmAD)3EPe_D`Ujq)8^)Ynp>>VY^`Icba3jTgWS_h#(y8he@m2UXr`iO4+-aFere0@aa?91o*p2~q77={ed{#y! zTOeWAV~Wrhwv5C8NL^4{_~Qz`Oi3e6&>0Kf6% zkhNwGmO0HRTt$NDLT^M~CH=lBmSz}aSvYA>1@&Q)yiDx2mp=7#^5EWH_ zew6%Eu_x&7`G|%DwF4xAY_X3cNDfSyIh1#L8X3u1+*7|bY$uWZksLM9j2Ce&0+r}_ z%@YP$U2=M!fW8ITYrX%p@c<P9IZjm8j~M>2o#eS z&;Iz!kJARHZ1%{2jB*!CQM%?Gz$Hc6NGIsEs&#KZH*AKsH)M*kF2z<}y%IJ*t(~Ku zcA;@uL~lh@5tl2{L}S{;WsuRAJH@wVmmlZvzn+w7w?or0*NexL-T-@JK$-EhKNiB5 za9ATTP#VY3f|w&r@D3V?-eR`#xSaD=8@=ckjJzFbOW}CXc|&{>tCBjLaZrvlHXO_4 zjMc~?Xn3ghz3&TKsF>F72ZmM}EmRR)i;AEz}rKRcNp={j2sT+X2H)kV(|DC1A5xDy0gPU7GxgWB+=ppH^*Rt%GDv4}QpthXvH{1;-gQK&}e05m%pM*J}2 zxWX%@^E=8gB+bLb>I$jPTBD?a~%OmthZ}@bxMuyx;HvO2J+r+xNMU210qFnK) zpzOozDXcq+xL2`sDG-gtVwBa8%|2>uWNT0-k&{L11}Xk!D^J|SrN~)m*r5h8+Jrf@ z=V?DBhi&Tqmy(lAu%3(3lX7xbHd)D+kKz}rG=;pUD4w0$L7*0AO#JnK0xs(q;&@`6 z`UpfM3mt>YRpB#*G(oOxQ9nL<4Zb+}Yb+jxL0B8=OT+i}1WmWTTLsfKE=`xA96Xm% z9c;HD=al12EsE|^9#%59fd)ZvoZeA3-y@+$xSquINHO1f04`D!OUyA*?Lcb4rEbc{ zPZ#8)5=K!p7`reBECmO zWGPcsKg(;Nm+CTnjbA!s!?i&e>|bu6-vBe0FsLCCw^sCxovK4=g1~LGy(E3$kTsUI zdFKG?cx>5NCiH9Ohl{$L?bv6X3^WNJg&**s8G?jRMuu)_y^wFvmF~i_4A%RN0n%}* zJ(X2D8cD(@_4R+R()Rx?bNRou0(H`y7Rops^1;Z28@3`lUUO&O*2pmp%XoSBZ zPpwCGb7dY_B+2F#%9M%p>u>!E#8CTlX<@=~ zfCUVqw(ipXEU+FV;acq&PHY7VkOE`i61NmKZj_-qeZP$s9iSY*ookoIePhIxfA!tg zHYKm8E2H-`h=t-+PMURzu|s82H!qQx>Kf=F3!Mgic;17&6gz}&A|On5+q1YyfNo9A z;Vp#g@cx?70cj*Hy+!2e0A5<1w0S|Y)16|>EgZ2qx{Uhu{4RRhOYkSod&pM!+(y17 z^E7rY+9a_4%Bem_mOYZSvL3CozD}IPlf*P0u-IVnX_q)a%c@xSYNqEp$R?92*y*;6 z?nSwTgoIr3a-M6Gui_D%gO;OQ64-{E z-`9tg3L+`o>LDW}rohWgV>btFX5G86TI`5i;imrtO>_ z79)1Z?{#gVs^r9_2}3cve8oMMufZ1Uy#1qs8FuE2g4rx_zzBLImCBWQ#%hQ~!n#;( zPq4iF3U?0Hv@rCxhBXpn!Ft!8qr@G8nj^iwwWJJ~DOM#SKe15dsBIwZO-i=HEhTm5Y-7&ECS4$F zW5iwggT)i{NA7OdMb|dLms)@d;qej`yB9y_t`CuXx2Jf_y*@gP4M7pXShU!gwx4dy zW(%T0Q}aM`?_+?fYQGjOL5V4%YOqt5tEO;qm?F5GB>rlvhjJT$B*z?TQntOJxO>!T z5Wyrn2ouMyw${d*rx21Af1m%}+c&abVarw2a|p;|f$`pqcQJ%KI4 zAZ0nm&Hnk++LSPO%|R=H=1`|Y^D`k7oj0oN=@XOgaQFF)HUq)x9G1p0r1F2$sb zq-3E0O!8@zZ!N@EeH9*%skw%GV}29a(4x7&YKT@#>>tl0KPmx?&Gr8Y^i>p55AyN& zUVN@O)T4erOm*o<0i(=D48E4|q&%kIYLhB~XgARAr8b85?UbTksNw=xr5c{#7k}b~ z0aS$`gF|TprNIYIoGT((pdFR3%i4H6{#UkGHgzcAc1iX5G0km)eB)z)r_kml_`krU zZjgzIj|8OI5ENO?)BG)c2CXtl<84wSyS6TZOdh$Cd_y+-nuZp_&He-C^HvHxZDbCI zEl>K89-xhA4DCiCvF~bd8}R{djwvk6({>*NNo1Hq5jh-Zr0Rvt-+o3fmg^WNfiw{` z4sE-0e8c0>+8H~x)#h6|lt{q}SQMf2)TY07YQ`GYm{FsMsrBlR*jepEN1p`BRIU}I z@jPAK0ur*cKYhc%a*pVqVcVDyQ;(TK`{VJCy2l!|pS*{?*%gJUTSTJ`yf`vH3C-&b z;VWX%MSpN>bHRD<`g+GLu|o6+dzy$nPX0|58R4zH1F)zJI|EP zM0tMYNF0B|IqHy0Ms&BHc-e*mYPwm`)h;kI=_a ztXcU1cm2^Qi%}fN29QhiPWY<{H>51Vk*f1UmHjN#2b=?S?>Hws7c5 z>rK0Fn0Q_sw{WCby=p}}vxmNv%i!w}g= z0?cwI5o?(&VzZX^Oo>ht`H&tUC~!qkOEuR^xgp<8bv+zTobhhDLb%UCGACCvH*HWO z04P84O{Yj;c7ek$3BRWW9na@TOn8X!=7;yL zBnt3`jXFueN=7^(=d80^-0mjL@6(iyBCf&B?GHqiCe27|9oEmag3vv8Qi}K zAqZJ)XGhBUcVusV$E}4n9-r#>ry%F!odAA!*Q&HFVM%%$P^Z&IUC^YsHsw6rN+1%v zSP|hYxaFM09=3f49}%5@MI)b?NmZtX9hC9o+=GOOkjgO-r#vQ5mqB4msXP3`ysbTjYQO%CLJvAz8*bU`LS&40{rhmsBf_D z(eJTvgbf5h>lc`UF=Tad?`E9cdTwwqWpSW7u{NS&k;SV@dnWX5!`QZs@jP`SI*pM7 zZ{M0f{Ax%YM2b5mtPD`e1}Mw$r0`km=rCgP;uwF4sXdfH?j3=O;YqbWeH&AD%8huW zFEA`diwT~@fSv)MC);H@l z)1;L2dFPE&ySjkf)Z9~;uly(c z0HOi-*{4x-5Jg{K&^_Qb=AkKZ@|alulL5*OYfZEtGVb@HS5qC`W-Y#~j&jbOrJ`sD z>UQojC@8dHi5YU~f7wY!w`ffjH$5=xR^2tI;_uflMH$T9Z636GQT0$|bj}LM%Qd9v z$lmLkouREa!46M(c*;`48$Xb-pu_vp*9Du+xZ+o*j^Rv7uQtf>yh>5y9lJV@&sE(` zBIVu)!3vIZ)WBWBhTw~qBqCq&N6cq3!MkHCRX_U&$eXaD-ah`yk)Wl zyM{BrBDl!N&GPw+aK%?k2&m?Rv!K~4 zTI5&7sC9gA^3HC_3N-D)K6)ydIjk9Ro+MGU8?@-^9^GdgpZp@&l)l)# z6_fll6PIv{X(a`UT90E#&J zQOS_y|pFv2?{^;y`f3{)#~3;zkdj>U&GGcxbBZAk659zO3U7zW=( znS$U%pBFc+bIJ&t(l!Ht=8viTZ|p1fPC z>?Ee&sOM-ZjTV)PJzwHMCj1ZiV~q(qY6DE z@V+}6*{V-zdE0?d#f#nwz!p28u9rUD^>twWV%^BHN<&9JzAD&8kMz)CE!#Q zM?;(=o<`Z9R~l!7t+}90J(BW{67_a}yc!2EvkH>3UX5V+mS2?Mz|(jhO zE{T$i9?1(%08w`YGSq?yZK3(L>-}Ol$EZ9>rv$mq<%gU|)U+<~`K6VE(P-(Dl;rHY zh|k4c8H4iZwQ`eh;ne67CUM?)JGNbexF$xd7KOwYk0jB_ty3hA(VJ5A8vX-D7qP$x z<$8Jl$>@WrDaqWV3yiztqvd7}(#?5=@ zeAuO*eszj+Cy*;|!~U@eB9J`A1U9?HXNu5YIYjfpa^hEDx0us|TL@5O&SK3bGyb@q zfRybk_lKIC?~cW_KdP6)5pC~Gna)F<3PfonWVK@(W<Y`3HH6%} zD_6p1nPHjhJBaLn+gpg_0M6}f8cBY2Vr##O!4|RpyE%asP*Tiv!4IQrt}%*=zozmU zxQvwX%(w26`FoZ7bEW$x2;%(q4VZ8_jZ=nM{%y!$K;-#ejBBLAanZuiLJJ;rJ&$!M z^=~%K)|*DC!1;GaB2ZeG-a4E8&$v9*QPg+a5DCixRAv-{Zc)n-=Fe!7{ymC)SCqI5 zz4ZCFT#Tw`8$a=_kyEw(+1ZTu>z?Q?-*&vCa~4e@r|c*D4j}yQlln({sK@;W55H8K zH{}~Ry|nuj#OapDjWr4vGyX|wNS#4u|Qh&ZqbcEakBV$8^Y!0Be zcmV;uiqA}oBEXeEBy(f!gNOyK$mw*kn4?Gnv}ut{jMyu-rYFTX;}4%M^`~a{129Wa zJJ1RMO}47D_zZTNcnoD)xa+$J(lpY$sTgl3N&1vq@f9n_57S?oo99_lo9gtrqdKd= zBVK;i`j#aUM1516BDXX~j0o{sb{A0+Zr_vf@kO+wxrxsae)qVK$S!sftf-nDZR*%Y z^JM8}K6_*(&JYpb=S?RI#)_*-d=U!S21WDr86udvBF=+TvmRQ$R|g1ox&i5?bO{#^ z>%~SRXgV*ZaxPXA{U_9k4hm3}eAl)#r}9I|6GEr+O0%IuJJ>;F?%Y2r)d+Opm)iyD zAZjCBhjZ{wb0_QkHV-ielP)j6DZ)*xCPZB&>gvP&iVi=coB*)rxqOae)%= zEK9lA;^TiND3vD>LPavMPWL`|iWr4JY3zKLxxZ0&YN7W(f1&^<<{vieQnNo<0nC3c z_Q_E7aL+wfa*Nt%1Z_TTy>X6r-sSofPnTNxx|&Z293i_=^kdS`%37CX7R-*P!E{4N zvd-eGk%jd79N(g}|A?SOuc|vstyJh5-Pf8`nh~Fv?AV=5vk;uq-}EV>6aBDVI68w4 zQl;%rML~&62M2bN5Ck!ZtkU9IeATI1E#X58fOb%leE9X`OKo=L*ed5qgVE3*#OVjeF42b zZ1&{MpI#~$ew}-@@?fH;&moR#-&fucc+$iH?UOnK*ORqZM}F!A;L-Q^;S7u<(bLp! zu)oR`@HHJ2jp(Gl$;K&%(+t{b%u}r-ztUq*7dm!$P1y1c8#^=&3hL8cJm5uSb3-I6 zXM0;iKDT}>o5#9a$x{{t{%)tyjX=&iQL^reDXApB%CiAfjoaN$Vy*Ljhmj&XWYIE} ziE4yza&LJK)m|Lfuv}wBy`Xl6tFsZJU`*!WF8TvQQFJ0SuW}5G=5vh0QdRXG&(;v@ z{Pga4rXgi+=nJPCvMN}Qol zb`D9egBY}wHsBeS_+lzQ(5@r9<7^WIAGkk`Fw^XiI%St-z3fNM&EnW$=tNbBv)xrFGKPf5}%=jK{K2~`{4dtB?S^ya9xU5yfg?G%Wom9C+ zKPB*dMZwjrol(!J(97VLWTXsi-q~K*4x$v ztFfN)J!S>2@70=gAmT2;zXPIoL=-=R74C|R%oczyrc~-U_3H4IYrKx;x25We1sy*l zn^DKbR!<>`t7X87%&7wn;k171`7QQe68OG7N)Dg4YK%rmim z$1{UULq*q+4?}$4>8JqQf1RV%6zq9!g#BKi`w7+Mz>lUE#njKiwi{W!jWV9NVkJ(g z_?6lwK)LivuUhY}j&wK79@5YFJ%f;AjcLVfAqJ@~ZDN{twE-Fb2S!08M^rqWQAeTn zbwF=w9rf;jvUSRAt3-LP52TC!3Hkh(6H1iWvWuR+?ItwHBq`i6XW|9vEAecu+w2ltY%`1FvB3V zSDJ3)FLX-=dd89p_5yBuH-gUXSS|1aw04U2Nh{C%B&CoC-@6Hcgelf9|H7xfSrL+@ zyIJ;ZA2>P?Pv7+PYbPPmrA37iru3Z)G>zd$YlswUe51+EJ0X;9enHw?p2d%V81Gcx zMfWF;FQO7EgTk&Q-j&x1uRe`#Qd1G!n0>bGV=4cPJJqCGeBkH)XlDyfCpV##*Y1^9 zLSBrBH`V-w`Fopawn|s!@i*^Z(IP)Abjuy+XpvN&7DljQT4H&pk zwF6myNYrD#9S7`!>kf6i2NN>q-e9C7wZHLzUr%SQ+I?h^$q6EPwB{3jaT8xeG1m$O z+Z&4(iG^tnHQ_zVMWh*UeuCxr@q$eAP}4L9ZZv$7D^CzgQT~{lsvE9dUL4FRx@YJV zQp}fRUqi|`r93vd06)^oGzpl0{9_Qm)|LovlqJ+)$=9$z%a$w1$a7orR?qC@yl5hd zksp%AyI=2BrD&O8uo~5B9+y>lIQT9Zm4+b(ng7!WOs72(83yq})dqQKV@0IG7;%tV zr?T{>##yg2{dqxyH)$LnK&X{*4SnJEkjdL0u_MqHex>kCd4oN5eOt_}WO zsD;T&3%;K4i}XgkGGxE!*P01P!ujzS6AG)?zxzyB$8EO;$x$#%!wdqj)G>ejmG{g= zyX^L6TFu44QNA3!5i#(bta1!3YBnhke%P0i7RW!Nz$*%~O8HEqaUX$JfIz+&f)}Ly zqyx8y=<(t!EsO-(a!=W;X6qShiv%7Ut!jPv)D}ctF2aOsTg7ds)nd0%k|LvrDHYzG z=}He&0VQg}GCT>A`G>iE&|E@neL+ApkHO;A>`LuC7827`n=obXOE|UWsjN==@(~yV z#HT<-Y?k$N%)0?~AMH6*<@+I8s9G7`QOa+= z7e@2)dm5t_EK?#ZVthH%zBR%`GOoM*S@q`vKMca=k+fG@&8sCNsQZiZdp>3J zH~_zkD{LkC)^PK?<>H#tq|-NVr|Tqgoa`Rh8(`#iyvZ%E|3uW!c#@ocE_rE;OF=c@ z6(DYXa`}A|I=a7SFh7Zn!$AWzdA59-+>S zPvr}i&1uwKzBubbi8&fkxK%BOtP-Q3WaI^r571U;ez=qNaxjT%d>=@`0xyn#!*)IfWe8aqzro_ zVmLo*MNKZj9X{3lT%_bLWVcZ}P2H9WCWejnfi1&|eN4@HpjX?yoJ3F>+@R#L(Lia^ zvS#=c^2Cu?>??Z@o=tZ-T0~QCTnQp6=F>_VvEr z$wk2CM=y3vvkF6ds9DRUsz|UGVL;!v82nHOTM^-3SQ56)p`92P;rF71gGoZ9pNDYa zl(N;4tGm7>iHWX3U4tZ&Gr+XN-mXDs@x52TPd3sB9J{=-cNBWk9^D<7M@ItxhqkYZ zs-p|K#32NCcPB{T;um)a5S#$P-66=u-QC?GK=9x$7kAg77k790`DUJGt$CQ2dFq$$ z({=jvsa2V7e(JI$83;taAuOHkwy9~#taL>qjeBd8sX z49Tp(w#BqCtc+%Kr~0!lu44#66`ZYBSGn=ty|k+5lue}l@x9o4Re}U5suu|+RkT5_ zqZ(S7E!w{1-u=$$#w{-yK(E221M`ujV13Ri8>`-D`;r?=QTb;(U(l8~V+9KnDa130 zE{;5TSab{fVTUp60f_ms`@{0Zm?0`}E^v_lZU8r=)E-P-Ca1K#I#jXVY}Ko3agltx zSKyZudwXElI?M9Vc?F_Q86`W#VBY}CFv7rm3&aTZ|GcRYnY`#0u@9-6S+-PKReU`_ zAW82Sn(Y`lbT!2xPxS1yi_48th}H5ZW>L5z5kbEcZ1$heUAz}xYh z*m$~l0a<#|R>lzlNiy?I5xR^p}G{w7p6IG z&Vr>WqZ7ETf5%NFBQQdTLJJnVV2@Oj7oTgSovf8~V7hrjqv7+Bl_ z;=c-^)?x8Gqw{P(`{ng%Tv|R>q7hi|;mVFQ9Piz|ICVuVg8^9E&B+==2>DUtu>ft^ za2?N6?TAO4R-SK)41aYU9p8xBiz`})w_%<=lWZrYu^Gtym`kG>_eW0VeqHl@XG&-r zbmk3G6>Adl$;sR=G)Y$~hfc&-mok;IK5>K#EhY)(&oH z2|UG%0t|4KZ!=dnl&<=oS! zn}b=yLvWHEPGgUMOj?hO8=S=nvnex*`f#woJU(YFyH()# z6MJ_6l+WRWh!ZmR4ThGd7Ph46@he~O%KtV*Dg0gLcyf6Xax5It`OhS-8n5WpOD8(I z2K?7~4upCD$BGEFcFT!3Dv@lMRjcr7%jLuJ;e|k!kDr5kfWxz$f!77jUu^iJlZ_h#{RS*YLq_J6;9A#}woUgi0e6|Ky=gw3DS4Z0!7Gc_TcpMRt$?FhN zRu1(!`q|hYn!)PGiDPM@+IZJOciLT$+JS^ZW>{N%17?0){Yl-W^)4rT6z-Y${0F(7 z>yLjUcFB)<&FA)PoV%(}GL@dwXrC3QGcVK8#i6eoaOYWriz$_DIBy9}?WJ=7WK>dZ zB%UbSMqFLgpN0;-qAt9pPJ!_2}pZEQJHwRehG97ut?U$t2rj@96YKCjmbApp|o z{2Qkm(JWv&Hc#dVd10B@N7i5sgbH{`t{?~V*Z+Ws-$n}QT}fAu{OkzJrAl6Ji^bKd zT`9wP4A&mLBa7Dj+{TUV$Dx~XX?R34nPc6UpaK+wQ z-MlbY!UO`4D+}Wz{lj4tBbvSZ2l$dGB!}jA4iH#hMer zx?rsS=fk(XT|jcd^3P6h`kWj@e`2{;_oImeCn!-(abm6&Kb*A zv#Ua(i5%R1kO(t-LWp-3a#DXZPbnowC~~oRMD$uGJG6+v*f67A))yQ0n}{Gfx0Hk` zkL1c6o59|L-Zt|J3uVbxJJU8@6F!EF8pZ*kxkT`wwU8cJ6)ccR;}-+zSgxHX}z7>yidA1*2(Kd(U@SuWI7+7^bkHacW8 zu{W+*?@O_t(OoSvOEjCnHYQ;dd(Pi8IC91t*xKhnG2lS+w z#vW#(z-G($GNmDDuGl+8$L>2%AATl9Wab`-yB#=g_psbP+lS=3!I7Di9(bW_Ky|NT z37?CKaMD$ZpvFkf5w@)=D;%*KOoM?D4dq=siX@5;rtYVP7hJt4OwLDikvd6YkP?N+ z<;U4$Vvi38x}7I z9YjGfugSq%eWdks17{CJ1Smsu>$itl;R5o?AN!F;)=U=ZbI79uEqS)OK91N5kb%x_ zxuJ!vbw)d9b@tSisiZG2MvhyC)}h9=Z4VrFgVCQ;^kF~z5_ZMDB#VT_P2(t`FPuWR zD#=-lRq_nq^K)@6rw&&?UP?cwfd-t6tR3MCu4Q&`Bl^!Z6keA+_5!T6 z!)&~Q&(f68^1DYWQ>Rc7vx1<@blic3#u%h-$N{wZajp)d=1@zJCnW`p&HZ&Bk=uB& zKcM}Ay$maA$9u`dAdBCs>&&@$bt6}P=PZjSJ?a=$;0)m5Uvvdh;X!(Br;xtz6W&UB0Zsq5d_H!svXHUR%+-- zY7SH=C?%xHlr3Ca{+f0!E#-; zI}+ti$wZ;kZ}XU+bHEMDo`G#Ytgpsc(5L96mP8#;+V3m(iz+wZ+^97k>%stjDl_Aj;kqpuZoMcX%TC#*?9V5O<;PWAn9b%N zSXCCWBX5}3dQv=PVM_yLZx9d;7+MMVBMM7LTWswg6amYNn`fdeO8U*sq{MWe^8s{- zQE)FXA)yYwtF#|ZUbj~UOJ+%EuK#&s&)-)1A00tMJ(IUOlcznEyyvHNRkFdQg+hb< zz~dY7c<(DXyXq~qduGX@SA&$Mb$b&~$eH~XnD1CvWeQ4296@XB%=(h|+ ztkbozGk2$}o4)=@k2tF-P)!G!W^DknkM8-Nb>W92wIL)fxo5Q@I0Z4)yQ?C<amq4&#($$*#u2hwoL zoqzrtHxSDFIz}AC3wnl+{+oZySC9X^T+w_tHfz^b9i8L)iS_xg&y8Z(g!kDz%BW-X zF_-zzGnTw+olVr$_!Ixgj)jQ?P53c2Y>#f8eRh?P(&sR_@6TJr6bw>lw&iu`6v)Oi z%lZI0f@_EcuzXgvH*q)q-8W7MMlBQ{59@YZC}vRV6VKMmH_%`&?ixVdQ+@%<9|)!Z z?g%e&@U^mVa%}0e4ZQi>EIF&XcB>DX%2{vMMYsAhr^Gt=>^M9@ue2JeqOvITB5kq; zcyfMnFm&3oEfd|`&Ua|fm~KQtDdGWvp1U6*j+5&V3R2r488$pWw>slBgpsw1t#e63 zs>EDmb$EJ)g7`v9Xa2af^njo>K?CNmahP!eFM}?AIwddliQ(CK%J8KM< zE`Lw;AQsh4Nj7`P3+{yW4QppB&8i=O<#uOhr$=L%Sz7J8P5P#T-883B)H1X`yY{-> zhxKS9)I2KRJ|9{Iqo7gnkanG1BVjk3ElCL!vlt4}LgLh5#_@VSO7S3pg8WYlc!Sqz z+)ky^+C?+^X{&fsb5kK?1`1YL49T!1fwH;yB zoosp)prwkR6gJ<#6pyb2DC(4(ak7(e5PhvWLH!hDiIa}fP#v)8fPrKU4PQ9TC}^zF zw$xxBtd{w863-U9yo;tKOE6RD$b`m3n z;9Mjt5ddTMYu1a})^APkitf<+HJDFo>2X&H7PbW_+=NJ#(|?D-M57m(-* zijAcGoLNE!b<5#g#Vj+*oK=6gh5ZuUQ^b?4;>7{$oDsK?PN9klDv@7UWICH)>e8yU z1lzg~e1Ctl4p5IWpoE*SBC^V$opHP_6xHbT^3$rC7`_2; zk0R`LR|Oa`0PDH3;_*uqiLLdELS2vGR3tfRrlg)VwSxOBqefNY^-d+e+&>Jrh{4Ql zJFC^;e>?g#14|U&ebm@zgHsH;64;bYfa}!4%;VM=1PaHWsECw{!mY=>;l-eX00kao z*ifQJp2cCkGx*ZTkXzk3X)6G*dWB4|(s>78qBWN#5ws$wPofvrsUZ6DJo78uQ*xAs znzo}@y_7{nmJfJ5eKI<;>W1tVHn=`uxtq$GsCgbr^qr^>e@snoF0=$a4?x(p;~Wdz zhZy<=p^apLghrjzwTD?POoNV5VBAEC&}kC)r{(OMQl{M*GyD@m5*yVM!t{io6yeWk z&uy>QRqZwHRa#+chS8U6YFm(4JkiRA!5G8q(kwxT#=1dn@ot?cA8$Xt(A--q-fHNs zV0}vUHlMU#9C!aQ(lyZ8dT6pP!=hX62}rYS$4r5(C(s#lICL z1n4H4P;~tH=B(NPBR+KyZlH5&G!{@C;DOo-|2@Db^*5!S(@*6(W6l&m&@u*))Ip!A#x!MkAqrr2;jM-OI zCrbTz*}ZLHA44-I8M($6f+!UiMx9`|_xzRZw(LDA=kb*0ePqUwP4N7K>Wm**}U@|Emk&Se7P3{Jhsw#8kU}(`50ohcjXApxn6^5*2OM(=~<`z%nZ3@2QE#3a% zV7P=VC)9_ujw^j9x#V9~_wvQT#tuo*r~_+K)Ne~<{l<~RFP&*do2M5boxRf$d?u!W zey`3Xv#V~^p^({7e{9x@P?oW8BtIr&?Hc^Uw_oLu9UWRVSz9XAZXpZ7v%4inl5Ycs z$^0R7c5JWC*0!bpGDGB?l90Kx-X(2=zOK_#b+LGQeJmnpr7Dl2;n=5Rd2d=vc7rJ; z#%qaS6XXV6JHjPnhPW_TD`DoWt+08Izqc`s4ZN2j*y;%glxJt98ij<-ywP*`J1o?4 z#zjU}$MNCOt<~q)j+zdLFA<-&9`iCVBvJ)&e_bg9AZD?C(o8=ir<077TaSNyZIJY< z>Jg+17NezD0@~g2n-4i=ubHE@q3384F7Tf4Ac^xRe8%kCC}@K2u(4SLCkPSCy}v5x zbn$?6fSqrXd(lC#7Nmc{Zp2>#Qu;T?i%1_i+p=a2Q#C{vE&r8q%|Z0G7} z_-DD$rHM(?|3C%!AW^bEZNg6PDKPTGDLC|DqP|AFEk>%`9@`^ME~9T>SdpLhO^>CF@JX4u>cH#yE}~Q7|sE zjyH1N7RtXI+5S};is;(_*r-Z~_PX2X8nd{mTT(7AGA$h4RZ&y*a?^oczYbt{pYcWY z5MJg zFvT$f@``Q1=ISE{rs6h-Tapd?i>!~@!-0B!T;W5vdqLq!#Al%wN#()cZmPb@%Zy|4 z=C4(Em-R=z;nDRSue9sC4>?XRq}A7INxCXo6(IrJ!7eyL5iG`YvH8=T=21nE!9&JT zBFrxlN z+6mjALBZynWvfB?>NKRdi`5M$yx6;> z+U_YO7-z9!fdK2WZsvk$gQ2v`Nc#BwL{|CEiWnYn9#sbT3CdKz4T9d5-9UJ_d9qE(2Z-~1LbiS{_# zn?R~}yxcUhThFamA3a5t0D5ITFh)yd=GYX|xb2%Jfpf(iajmkYxF(iNP|fESi9=Cl zZk)-VTc~S8tn1LmPof|m`~K<+Ervf9->nPNsP$xHJbq(O`@Y7sKS_#A!!?BhrsL?y z@eIvSr}E6_67Is9Bpr{oCs$PQQw?l-*9<0EKs@Kr>vH;u5rJ)=(&w*a@Y>QPg79}7 zJ7j|U!Df_2n}*?+kqCB$gyCF$&rOBy0?L6mPG4t2;2w%eQqs&W(9#||_BU*|6fPfn z-`FI7sXi%e&e-NRP7uyz8LY18>Cu5E=Bs&j@Eq7qAr z96Y7-LBiIwLQxglncJ5MJ2LHa5!srnmxRIzQ%rt&vKYIp8q0f{uvzx;A^Xr~Idj=D7}r#lLO@dl3z1HzItqj1sg}&9#@L1M_ELxb>c8Hs-FD z*EXU#cw0WI>==e)M(>W-b^CQa(am@JOwhd|Hj73R7gd)UpmwYZt%6Ir@F%?avKwZu z&4JVDI!)M6Q3Io|n~5No&L18-bif;2I~juo7riqap_A{|nWOUd4&v028J+)5o#R_e zZmvpuZQQmLv_hPKCk1; z5a&GWcrHDbLD@`RrOK)iwN`EzW`kuI>a@LI-}WKb_GAqbOY@gLg6JCq(ni9M1?Y?N zjV*D|K7jj8guRDAE#o!#cK0qLT8WzU^$KC7qLRP$oQ27LQU~2$K%y>!JSxl2!hCoV zNvfA|MZ9XMCu@;Ru_|-zVPhIO&tk@I@f_VcwFz#HOFcS$IzY?MMqj25h14p&xTZQDM1Q$6nOG+r9ko!wp1xu4WJ3F~z{R)rjB zHs1XB(YbY9{7Hgf!ELb6(4go?{QR9IA+3M~xe|9)=Ly4V!is^yh$OND*ng8r;kUJw z1=*CISD?yyAFHhTjqOamXcL!yP?)};XtoWdcgSL|z%in%(BtQS(!$89MOjwVTOX}O zAF%DI@Ila2OavX!xP%uvB!pNZF_p&oNBy03IH?|vwxnmXO@0%M((r_l^UVyM4rd7K z2`f`}=Q}RCJ^p!BwIw*f{+hd%CC2K4814Ck83&d({3sT|8>s5yYv(e&a2l@xWei+4OXWm>212Y}Q*;$t5I~hqi56XKuMTemLglQrli^KMQMe9F+`)!$y~3NO1w&z(Q8A%(aAb zexb9NG4LI^rEygbhrkjAu3*(!l%9FKSe^X%6X<+^g_f|&k7rA>NYg_dt{KESA09jI> z-IzF*y>%wL&_VouKjlcr0LRDnuPVhM6gDD}VE%q#A6FvL5+=32oqz@{h*@#rC08-pizu8Z>es z`sBSJ?Og~vkp^*x8AZ*HovH|vj!*1wZNukJIzE#Zc8=D-M#+@MaJy63g?h+Nrg?Eo z^o%3o%+W$)>QmA>eTgAyo+l2$vFZ|8amw<)mfS6HzwHGt>Gp;nFaLs>Bwz1_R&?Hp zbGk%d)Zj?`A=}&rL;0V`43~){)-U-`d07ZZkaN;GqI_1b5$K$>m+Tsnpp!HBe&`a%BTtr>#BCk(uo9ZUENDnZ-q?+;8BVesV`%;Got&fSF$yK5o?*@R>)~LcD zmn@;=k!eO_{y^dA2~_kPBL1gGoU4UFfb%4;yg{n|EBZ1%#wE-)JMAZzw$!nVlB!XC zDudJ0lv!l2O$a(#Q`+RJM48&|?Y zOGKcF3uwsrPI01u?^Ep!0+|d(2Lk)}xYPnC_lQgs9U2Yh7)G;~3^x6(GuxZO9U=!n4-Cd-=QE-L4?%g#Edb(Bx* z7_ro_m9)fbKZLFJq>p){3n+pNwj`JpI4$zxk?tKZi1IL77kt-O#?!aJeZ&>3^-^x& zI2AlxtItAKgwEPhM5fOoqJhUDVwT-yfA{YWWC8@L@Y1%{HeL7r*|28ah+5waYs;<} zjVJ)e{#i4A8>MMnctSBGaBi?K)f`0;H>AiR&raV)kq4=IWa1aE%#@3vCy};Vn}p&? zv2Nr3#hlpN&y&&7LnZR@p&V%RpxX~099e}kQfjK6%MEj!49zdG&NI+wl+9~P&L;% zc*}d@KcmsDWr{X@k6%3r`JCiiXue2g(#TD9MMiT~BS&~&>q0lBu*Lrhg#(SX^_^6L zv-EY;Dti&Vx`MaMNzs~hKdD#>r1QKU( z_3MgC;-}wUyREO6?CbI+#i4C0llhl>;ks;3;5e3pWYZ5#g0Qhtzak#+_WsrpEu4RP zL71>51~l;P__DKp&&s-G^HtCuw3dYMf=LICpK;4o`w}L9I!#*;pjvFqw&DT?%u7Cs z_^>E>Mbi&C3GrFKt1~Dy)-N*55$Xvh_gntisO{{|cH2Y*N241W(t53xGsQ;NT zH!n1NpAJd11k0+0nOGiwBpfXl5Qm~V&$0D3sxM}`q*!FRXBs+yOY$5uazy+id50vC z6g5&_`M#%*eiY4eSk>M^)N~Mg?E#f#&8e-fcO!e=C1KPyV#R`Bhs>V}*g6zPgDPx( z7XA_1aGd|#+dBng#x%0B>fosrhq$~t65~+ zwD-~X&~RO7fr!HUX4z(Les%hzqJ;RXiWC{dJaIMT?B(5isOnK;C6=|JS(xi` zR0_skX<4m+BE;n1*OIuJ{z%`EsCnZH zWD91T6n#G!3jSIqXwrP$%ooeTi90$AeuYm}4aU2ZRw*m1K&GcyW9PD>1R&U2;(7B` zIqBGsDxGBuMDlD`lxh~g%fz6`UGHEMM2HyEkcvnbj@N6eKa4mGIvSJNmzmtZnBMmm z(854}VylmkI}xA>nJgsiOO#FhMT(zrD1W(u=RdL?FN{*f-MR;t2i+rPI;!KrN`2`W zq%++w_ZFGVQ)q~UITx?$%}az9)Ap4QAV(N3DsTAHR1$lPYj%LF68t=RZtzx8cTtdC zPc(u3GE_VDw@EEI7k8$ksrhSc;nd0iaaYqE+&0&})pw@*pP!EUB7Y)FQ2(*G7KzB~ z$v`Oa-C`Pry`1fx5l`%?8^o{p)S&@}&My7;yD<1J9_5_AY80|`d0Q`*ZS%U5it`ud zUNrs_D=K^ZBBdyt|4C8Cy@!#i{1JRfL0(Q(t`9C+uI`>{P>=>bV3`$ib`QO{^U?oh zOQ3cvt$LR7dFBNVLIwPsDrQ9rA0~s>4Mti@Dr+XlMa2W=0NaxK2s@ja%k=XNFvqo# zDkTfj$`iEmiCf{QYKE;gX8lsQwN5-iLY;c5lt|o?n0=n>J%e>+_j&90 zeJTr0KUR`1HZl1-umvqFzKZI<8EUl2~7E~)YJScDy~cYa-u z*l?^u^S(bZedXWq;Igun(+asKS!@_ALzrj!F|*2d%$D2hLtbvSzF*9>%LfgSb^Jw?Df9VK2r^afS{&@4C3_mzWWp=VYcoc4N&aF zGw{(fns-<)2O@;mT}sWAv8x&zT-%Uj!s(P|${}7h+n=+q3rHY zycq!ZH5VPJ?Q?v)USsB`m2TZ+)LOsHY(*x#n4r)E(qs)t3`t65XKg~k+taUS@%cqE+=S{#RIGNJLV(-RlAJP^| zd;xOjcx+^FLW#BbLq=7>gOZTrfLMcXgkxY+dp=KrI8VL@{YIeMXc>ZzthwumBchdL z0aN>}R&s~ao}n#lg~OqolW_GjkK6XmFrIm@VC=(YtvOA_Y*aL#YmBY*Uw=mq~z>2W&zcNtYG!ik7}~s zH$`+}m06GtTVz;n+^to*YKa;Qn$=`>XOB$aZ*S3yd(HPRh~$W;6=)3L{^mygEtcrM zYrulmXmq6qmh3EQk0xpem>xAp^E?~9+#EIzt4NuMF@WV$$K4>G!G1O82Rkd)zan9_ zXohS$$r|pufA*sY7`={~m{?RE2<}?N+^z>)cOf^?CZl13e$+ZyKb9U$LQp>I`evIK z=`xh`1>hJgkeJ6RjTAfX?v{;1qI%;aP=u&bk#e_nj-ZlFz^!80{bH18$ zgBMKO%pWUgVO>Atv3#K2AWEqqw>r=*{mEuFe z;h$mp2ZuUz98qWTfZ-_EC3Z-Q%9v@Vp&`Y-OS$AhxywEoKRQW=PIyBS09zVT#n~&{iMlI;RRR3sP8X82b(8O3e|O3w3D}3=0b)mw{>EQ z6a!~Nad0k4--*w?C%aCxfX7D)Z>cTrX9!_5Ak4R8~=ggV%lOfq&k~slj zJgx&ebY8XZhbc1X7<*1rew#-R&wLqovA*jyPv{pE!)u7sU9YU(yJ2W5mevT6UUwi zI}+E+1!_;{A#>J{pYbC9j^!5n18p<-mlmnh`=gIgZsu-@7ryJF6DWs&rAc*FwQkj$ z4LRsb<6CcwuZF3&jY0@I;o@1Uq6;N$L;ot?&wO#NRAUaz#E)urTU~F~zg#^ui0kXz zF|Y{R5-!_KC3f3We(wk0Ju|Rt{fGI(I>ZBbmjeIhm1jz8Pzifna(35x$UiqETE%a! z>xtIzVw3T{b_nS&PyL>a>n99+XL27<#NqOrtEDhc{^961CU?YHN6x1NWI8{=&a0%wl2J!6?v}bIhh48EcTYf&crenWeXN=!A)9Ij`Sb$ z&Ze{4DK+_=xY8Jn@7(?7n#` z9XG6_(~2^LXMZV+g8}`Zqd88Uh#Qzcsg?c^!Yikh6$utck^tMc<63aeupu8 z51!5S-Us@t9;Dk^T95;??-;y*C>-4Y1*_)egCuri#}HH9@WI3Bw(yudvBQ+Sa|C7o zI1Sn04jT%zkz(21cHp+nLlCZmQNv8E}fm zaTSObTddK}i*|GNA-@)<6T=(ipsKX&<>-VwA-mN_PRuNcHeDVDVW4@Gz zrWxVC0KE{@A^n&7rn}UAOm{qb=VfZ$9Jb0H$Gpz_b}ymy6gRLURDE{?W_>x4ns3Wh z?>z*OgUEcz?-bY=v|NN$)NpqRRN|&Y(I8zn@N+;sfe9Xoe5!?FAwi6a0o&TX`w9|g zzuQH^O}mRKj0oiCRCODw-Z$r*Cg`;5rsMvCN`n+IQW>e*WM?*qms^`$olz1 z*YV#ofquX8G}7G(l{u-*m!hn9uf3FL2~mQ9!48Je)Zt&&OwiA8e`8yjXYmEnH5+yt zEtEb*KF~QNafz8Xlj;sZK;Iszh*C<&J+VY(YJkX^4ShWh4qaMAs{H0JFn?hGVetIS zp)35+%IT5rJ?XtM060YW?U#i{6MIAq+vwQlHtO3r#6JeNLPhfrbf<3^-(UNlSWi_e8Xxx@j#Wlz!rL^+5)(x&DZL?-tidJ?H1A9lV!UTaT5W6bj12qwra*S)| zlaey(P`+dk%fmh<&Kym76;FOLRPjNQgZO^ktc9nRo9Y6fbvHGnjoEXcsVdX^B z1Tf){**sJ&!*s}%=e;@`lz?a&@pzqcI`mx*0+F@g4Roqx&jylrf7JfOdMU8K4PR(D zl=+?v4_&MKA)hgr-wGU* zy!&9hL#`JKK*9kqh@9>6tYJebVIqB4f5)K8y6Z=(ha%35kMl{TeYq))5?t&P94h4> zL`Ew4tD2vg{f#cZ56kZ0q-8d>jhe|+=UtUSU8y#*dm63##J}=K9OT)&B&?B@NsLO|rmzL(uPsv*o?d6>zLE_8H)Db~)C^^CLz05w8={6vYC(tGsjQIJ&XC zye5ZcKYg`7ts}Fl)_B|>Uf_INx5#Cy@MbmL?jv01ZX8@w{Ps~g%vN!;e3=ruQtFC& zicjKa(p25>dvue6^}MS-Co}JGBV6ZTz4&OXrBTZTpj|=DZt>J+1P6`u@|X!BaS3Iu zmv_IsXR8yNgN~PmjEO5RzN3mP*_wgR$)M{kxA~J)GIKj>T%Rtd10B_nQYpbY{d-Tw z0`os)??QG1g-2np-;F|}8fg)en1yAomVZt7kVgOJ8)nGbA~h@&)X4KnqH+GyKn{CR z(lxv3k8|r8tn!Bma${-c&)NRVrov5CM8^q<{QN`u6jEI|TVA@sc2d-s{q}PY)f{xM zwda^L5cwqBNAoAj{OVpAso?MoZx*oUu0fT%g=R2MRxZ;&4hVcIN^4T;dU9Ktn0!{h zrI%+*yCE}Q_(;%nBgZC_-oyXV!!LeVp7^xMcw^ve`MmLNyGLZ+jJ?!8kq96-$yudc ze-YO4)|9g?@_tRwZSDBrxWe@Yu5KfY=KP0Q z@r?V>X-+iJ;i zZTjl6HqSN%r% zLsBwCAgsY9hj@_b4g5*&f`<)F!yY?X_+g<(U{5#+EFZZvWw;+Vxo34O`>bGinKo06(z9Paix|G+5h1Z93-_L{SzU_Y<8wW-&gu0)V zAZ7ijApIOA)wK2(C4IAx|KZsWK3aeKu^Nx2Gpbu}-~#0wPu@pT@G52hJ%{Zrb7xu; zmo)@2C)f0zqg}kn{a*yW_j}?UKHv@9F?yFD$55Y&$p>x|ix^zjm+X3BBY7|1ncjHQ z)X_hdjQ*~$2rZx>__1aIII+IC#Xz480o|b0H*xb-X2IZrYr!OW+_(>t|8`Cs>`Po> zy7RhsZ?UuMm5|h>(^VBH2Gk&41CAytumHH1Y_Tx+{hp?{>8tNz3PB}~v|!u^;O-M_ zuVUla=mi?)iw{5I*xmxIywE@5(VH|(-;!L?-IaGuCNgy28zRXrNw2j@1ianh^e(Nj`Y4h>j zvBBqB-=3nZByM}y5EsdLU>n5^NfLc?bUq?)TnHY@JJkk8CPW~l;N0JeMHnP{R>&M^ z0KQ`Tn0o}3j{q$?oXs606~TY`ZZ`}*-~;&^@O27aNHPS)4cp!CBYar@wd4OTe{}9U zW|6feD1ny7`ZlJOwbda(_BOj+{of5rJF>Slrhtiy94hp{gGKj`VD$WJftiZ1HQxBe zI@RF0jpCDjB5su_$o_)_?d{mj1Mg!)kX>Cw*aW9;2}rk-C zm|B`!I8$(NbNxTc;{30%znR*ZJ6lkE<>2Aw_&;TFI+Sdvm$UaD!-Y+7(#K?KVak5P z>clNiFyK(sm}gCkkir3oVd-JgPUZKzwO}|c)C}zro~GQ>0#S@>t+fWT0--nXez4!= z^%(rUHy}Vt^8Z2R%s#cXPAcZN#Sq90!elAO(8%bgilm}8LT6)8mXj+=Qu#e|D^&U= zzQ;f!?sE6MW8?gF{hTpt0M?5pfhWjmuLW`Q?=1qWB$fM*xXjHU3S35a&V!O%4-s(4 z9dQdm_)ZdqAd+s)HjfKeLbYIBz)fFnjBS6<3HRZSj2`jPu=u1)k4shvZj+Mbl;;{g z?cxKZxz9JRuoKUou@H(nRo+{(PD z(Vl+iTY-p`4lNSH3KVgqGQ+sMwv@j<30ABsclOg_C;HXp32MPPN~rO)%d3l(0V6(c>NDco2jJt;oIxq9nuMor)?-WN88Zf@`r| zJmkKXa{G>ZWfQ#mFscpYDS}psOBlWWOW}Q978#g$&mcWO6g|OBe|p#dckKR;;Wq2P zA5?tc(SRG>K{M~~2wnlPQ2};FZAJv^WeVH zmqC|eGTKywP6}$YXe;ah*UU7e?XU?Igqgr87(rE)?)uD`%6Q$7#{iVFzveAuyb|)< z^QtiXYrPIY!dp727BW64C$dTt**!GU*6#<42#SZ0u7)rb{sR=Q84NV@g5n3>TT`>T>po*1$ zYSZB5?{C3hdiMSe1vz4id^@+_yXG-N2l(p+n2QRv+S2kM8e2(t)oZy1kcnUip(e?X z>aC|VXLn~HQ5M%LZh3$48Qa>F7mIlobro8c$5hU@Cr%pn5Sgc70=0onj9BvaMb;Tn z!o7yWtgrho&bbN>X;4=N1hLVOildYNV(LZYiiZ4(v2Mgdbim(})+gnf;KFc%$dg_!xZxxEYsb^lTnaQEb5@3^+TAc`E9oxg6gCtDhAg4eGjbdZ zYg?X9dwf;AO<(iK{-sBw

!XmN{W|iWG~9)FxYHrRU=D+q;CkN?o#tT_-<-<*u{X zNvH5}+!IiOH)SCOkdmv<3GI^N4!P$*_YEXc{2K2lFhw^Nn$sS43hJV(6^1$^;VV38 z#+CfI)d9&=cu;HTBJ#cI*FIvmN12bA#s|*=)Y$gCj5Xu3ASzbKYK^tV!-9EMerKG| zc7_0Wsu}8&mF9EKaqGuAyBXr}yYZ%)=+PG)SY=k2=MyIg7Qo6~1n=lfHhk7V7n`m(bP^b^%eHP6#{s{r=> z)@*Em-nj_fil2g>lX_!4^p#C%iF*!?Pl>a4t?d${R5Hqw_!P^CLkWdFTlzyodT;V$ zQp*K-weJJ;v`k`j)l7S4k7c)zXy8Nc+qR1+FQUKn$I)h=>u-fL z_If+`PR5NrAKUJ|xQ~TqCaAmZyEZ&{=F-y{Q{>fVusA!olX3Y~w1iw6?QO`a4B2Wn zOStuPA+ETYm%*p153%%X^Rb9<6#c3_Dimsi08Vt8-$Wv$ky9yWZ*QG1l5kQMtdb zx+|Xrti;i~xrLUgwvIC8&suk=>#nDr*&dbB$hwZ9nGAsekTtBesk zm+x5u^G%JE2!2+5hi7he_nI$_3!>E`YJ8YE;@NL_0=e(bXOFO&vx+9+gwPtLqrV+ zJhU^D%g1LkVfs1P{K5pRZ4qX8li>hus7+Zh+`0(emaH_F`gtVOrHPOy@S({boHD5E z3U{7~X_4!nLb_rx2-VwfNC78j$qo{J&*6K1IY#IFspV4D_Qu^^(Wp&w#Z;pnz{>0* zVwf5fLhYoS_X3@9ZLNfRZ(h!J5}A1NudrmvB*83t{va>&n#}jFSB-x*AQ8VXh=4_W zWBaU)=p0oHT2$51L^)u7kHiyh{4lA|J-&x!HYQ=N7*V%&VNjGA?dxNGCib`8Ot`lF zDG|S>gUhM9{%R&ivBq_DIYTG3U+AB%dh-NEkUv|3Q9p04bpPs3GPR6pr$?pOp}T)i zSaDJxNVw8RkIySTZR^FgeVEh}X<}NO>(Ls9)&HJz#)~LoQAX+NLPi{wuH;UkU03a< zLMPJJ#cr`9($;W-mIL$rOzV%l_!@6gPB6*4)>6dc-U!&p=he8x#OPGGAhkpa7HGAa2+%N8Sg^=6r&bbUJK1W_mmN zv*meDP4*`;J6afJ7bQ$q4W>isjjbMi4{s*zDbV&uZQLf(ZDW0k&~u@(GY%C3K-)wd z_7mFB<)9+>;+6QTZn&Cseh#+*0B=B73xqrB@qX&;oh}0uloNwLk~;nQy84YOSJIUpDMw|sC3a7f z5jOZ=?kl6hi7!VGlR~%MP7}XXVx|NajCOMAH7wKGjYK|pA36oT)4%t$;zwOZ_dMiE zC@~8%24i98A^p#yCM6x<0`iy2r(_>;4{_fa>;wa%!I7BgsSf9SBlz2|gjm zAiwO}cUJ?6{2}u3RO9LsoCR;$+VGkf&0WUIY8FQP#F&${FQx$*J6YfYS5NMCcb-pyuc zhT;Z|de=3na4Mpj4A6J*j!)*CHJWwgS+;T&iee#l;Y5KoeXd{Zdqy**FS|gUTh%L_ zk9PqCXbLQc+@uNS5_O@bUr=x4C`FrI+t3e69P-P$+~5#`8~At}2G_6p3skX#dWkrmO1}-^IHpD3l$d_=K!I^4oQZ>Yr zY#5la*q2p~pyPqGWA71kt@}!QcHVkM_4if%drQ7ni3CsG&+m@>C3aX-IzHTs1r}+t zD6gqtR&&oGwRcueRZDdjQqpV*i+QJtbWQ3QO-T0M;pSQUR3hLP*ZqIi^H(PpR;44q z;VH7*qBqVPZG;+Qsh}hvXY7FpE=Oh^%j2l13~c5@{`?`a*Ec?H7tk?l9XrMfoqt4} zojjDHxGSkUXR&ap%WLsq&Hq64uyH%r1v=B^zra=LExh4M4L&n`KIi_tebSWJ`97IR zf5|*~9{Iy-YG|26+p@k;kF5?jvig@emQn#2l(FK^lg2xCwQ!yy`NAyiCcd)nX9t6* z+dCL^p3g%K`P01qc?spZs#=AEDEw-ggDwu!Ifk_vA_Ugd5FNe zJ(dIF3vxZ$0X?z9bcMl_luU~HgKVj%u{7i#T|8vSos;?}Fr#Sn;BmvrVCHA`RVtiF zJt5t|CD5-6a4l>qA&?<|*>7y0&-oDhA|hY*TJ^*VsZt=z%Hvpjx5u7)^mN*KtM^!9 zas~AXMnT$uw2FW?5G!g#u>+hrPP{{*n7P{h46CDNd zp_rn?T6AV|)5r$wGRa-Y<9Mz-_(cx}Fko3^^9w+QnCwbHgc`fT#K3R7qA~^Gk{}S7 zzn!s*KpMgJ%@{GdpJR@+;Lv%THRnnu>~sk~jz#Wf&_4!Mh$k9dRO<;5WpB)v~FzF0UhiYgG6W_%p)SyNn~cu z4iNIga6vj7a>AT1p+!Cij31Wrjq5{+n$+d|6oNChChNbx*~4iP_uPBT1}1X$SCz+i z%)kd~Y2X*V5s6|?JF+kuguDjFq_h`z?vM2}Bc6C;gFn7;jwtBmv^v zqLLxb{j*R17F-J4VV)*!U(nyqZ2agB)lDT>X#T-YAWN9cEDQD2gA4|>SH=}6-E%HC z6Y*WG(o{IqAep3{M{b*al(rB~MX*4^U2e{nwR23ViViC1d%9E~ zq1tfV$*_PA+1r&US&NK1FTomkL~)C;ic*OwGv+ADlsJss#y{lu6Ug7Ej^5y>gGH$z zm02eynd2heAI)YLAz*e$0WUN4MVX%9;i~uyWux|h;A4K9D3VG!ekf4A0NA-fMcbVM zrm%Ued(!QqznL_FQOu;HJUQy6{$V8Rw6E-ccrWhX=K1XY5Zyv&6UIqjTO?4IM>%O~ zvnz&$>=Nw63RX*y<{XO6gkb*AH`1XuO+1vWyo3YYvE5uC=>%>5fwrt;{>Ud4Q4^{v zq{Z|mmu_BJuRu6S^(gtfBWrN~NOgw+i?TDqOdDBPB2`MYHs@dxi}CPfjx9IreLFc< znl=Z&Nz~_iWpKTUaWuUnH=cevoJqy-Q$P{D-V=;JzYf7c# zJgaf%?q)-FPr?S9;t1bJe784WD(V})!aVWM?Z;tm(5cU|DU6t{tV$&{D@wq#FLl z{BMi`QV|O&hgyIe{{vQMFBJ(WvAy?6j-P{4xzx349`jny;RCUKLA)eNvD82qzfyPk zq!A=0KP967%o?7AfrsgNlI*2wos|;JIsvrJGaUPUQKs4YC=J%>O_iVhC*%;xXW}ns zz%WUp*uPr)Ba3$~lJ7>#nc;4%x3)n5!a zX{|tg-?n^K)ygMqLa|@!6F2do;eN0rl#_?p&G;LYvJYb!bK}LmN)oDJE|>?V7=gs@ z=v0Ar`Pl(s+(MmW75}bSk@EagqVY3eVZwAmAg3Z8w=%*FE)17-foM^_fqH#WvwU#D zA`uN&LlETsb+AvnT^pi@Z7EK#)gJfCTR)v<1ZMCz&|(uC>2>ha8KHDM@a}KwAm4|J zd!D>Lc`V6LfqHh+(K2R=$6YDdFgdZa^a)vSvK32Z6qJw69~8e=I$&vINW-AVCx3#@e@wN8zhdiD+u$Fo+kgoRQ{ zcI=er@*MZQ%p?M)xl_^O3kMmX+-u~vtC%9xrVEP=S0LCzugqWuh*Wh+re0#d7yGjK zkEweeXLTx!gFvcCQQ#7=b>gS+6jUlKBguR2B?=0pC_v8ng>;W;CWW@h*6*t_Cdgks zyjhbQq|&H0ebIw*SO=+)W@8*SQ-0xr!hNOjC-Qaz@X#%-pg-KSpD9%9xI?#l_Ws_< zdFM1AAIAbNx|nlJ{N{PEdfbTD9-<456~&fWYC5{~spDFXuAxG}f>ml$kB>`wXk59+ zKvPpwRella9?^7UM4Qs-wyoG}=kJph)cG;u4zsq=S4+}I{Q~@x&>Kd-*NjO|_XAP2 zMffg$$9rM~FKHUpN36uXB)k^3l`)6u-a7Mk`vNyIDBZO6cayp-vI?)#KjM$$sTi_? z7;F{e8AbaYG`DlKUQ6pmTzk(_6kldo|5AmKP~|hC)ak-XhYFX0of)(#rEhB@MpA5@ zg%IK=T*Mp-)Z-96bgBkn;}#tE<%VrB{>LxWE|$t|Gq+1KCIgZ2gApCCA- zNCcf@>)^02AnG4^HkjDopE}adlVDa2EZWPB^sRu&@1f*Qsm<`&?a=-e% z`dq)BSb68F4MxRoFMDXMgE{k@lfKQ&u2x#4{$lx{{cDkYvkW(@G^-Gy*;xL~#~bO; zR*Ki$s+4e`Z@Oy>em)!plaK^26_wNg*-hnwsTIC|$mNuMZZAX&hX={)@AV5`C|(Up zmhm|dxOGP23#8hi=ZgU!^A0Erxr-wxWin*JlCt!L#Ym+RR3%Sv!|=VsX>nT(z^u9` zpYwC~!(OUCkA-7}lgSh7amvHQhT0PjDQ8EhRoh5^gcQaJOpFxD${jl$+z;=Yp}xl> zfS!1ltXRIEe)L6WKI~!k{<|j}mJ=SwI7S6hRdF1{xePzm;2M=bN)wboU<6$lURG~O zs^{m*&jGq6ckNsyW*~YTcxfKii`U&kuk@Gy#T2I@(5ww=M_~MDpF;Zz}4Q$7~S%tP;gg!KejD!78D|}dlU%XEpG^8 zfUU)6%C`eu<0N3vxJeA5R))>A_d?oGH$J&^UeRl z5l@7>f9e$o8rOp{d%Ij-oc}Y{{`KZaq|KR2JwI@%a`CSpHY~EJp61h#e@Cyok)Ved z${*joKA2Lc*>eZ4WuKk?5Dr0Wnj-f%_dd=J0~EG~QuKZ3W2WRupfTef)7r_#-Q_e| z=sMM$!!p2Xu`okyQWPhyU#C%OcRzmqMR}_YNHOEs07=J3`iStXSaBokvG4e@3;cWH zWlCjBa=|%#sTcmYCaLGZX|w{kA#chu-gjJyS&FPumA*h% ztT<}U6K)eyuPVln$M5*hDvP^IjDrWBCtx$o2zZ7n@gp{?1?QYIzgMHoLxMlz3FeS< zzZjYgjctGJ@QS>8e4}dMDmrTHU`Ao7nxSsVACwEiSmn7d!KohR+AcvH_WR=31SXcE z!C&DAs0xO!+wXoJ)*ghk>LzgYNMoVAw0r2eeik@W0!1(;UbpxA!@LNnx=?UDqnDQB zDT2l2a{}`6&DYvo`l)+K)vyew!TK9_B!CJ)TO__2@EYzoRNx%fQ=R>6DuZC4i;dw` z(4x_)tc9c4C`9vvi~u0jH`7>t&YGfMkx)rB%Cs{*PG4U`kT)UJat2PMu0GCHFG1rc z0rtS8!_Q%xvc$4%j%URgzq7%+3>SL({*iV}7UJ7?#;pZArauePr-qg1{2c6oBGRix zDLgzBqM&bi^>&If%B=9Jqziy01=g44V~20%yia-2gcU~37pL|w;E&F48LlZkN0b#E zN^;@A2c(lkpuB!a?quW+2skFz(q?$(Lz$v4_nR5yc>Ig9kuh`@0FgZF$yby9X~>=! zywtfx{`ml_UV?R*S!IKmp#;-ZTvW9g_$)x02*0Q*A@YrbG>S$cAhEo>2hX25s0$)Z zt>~ASZTEL1;UA$A|6}xzOHX@QK>?vqSDcx0 zL@&PbITD|xe3jM92>go;3G_5XcXOU12A`N~1%EwAj0memq2g7#lpwrHQf8AUyeg*r zxCM4hnp!d1K?zdUV+ii~e)l&QReE`8J)W?!%Bk|HBEjE90}e2o z>a)xz(S3tRSGg&)N`k-#} z87@%Yrb%-11OPXQYw615UEKH25U0;S90hq5FgP3NI|p)c$)_*P)8^O0Z5V8UbFB=;mU5O zYea$l&OT|=*~yfd4ZZ|r`ZlIJ11^q*Y`|-b_>FL znNra&>5*0w7{H9Q&vY}LaBBwTMjTF8z2?&B_iYioi*K5efPhJ<>FhfHXSbJMvFCQa zk$KAtuN&uS>1eRNP^iljF0+TzAs3aCY!fetV%FUBQ2WSlZaK;-;tiCs4SXQjc3CW9 zsNy`#Cv8>=z+8E>=o`)M9atGB5d2w!d<}l{=*QI)+#A3RE7>{N{T}d!9$05 zh8mTR_JAg0K_vY^5<|=)K|Dpfd`R&?A}a-8n9*S)pfvp`AKrV`J+?n4C(jYj?>o3dyjd30UGVedQ9I@-l4@o zBQPq(;2pT=hwMG_`x5}%5m|s4k9Hdd-sp>tWhH4jP%NJ5OOT<009j6hrYM1aCyf~| z<G>M(6CeqYPPwO@k^uDJY&$lrBxiI;?;E7Ye+P+S$9r_6=N~XGuBDRXoKm*Oh2b47lIS|1S!@Ls63@cDED%%K)grT9z5*on;%qy@c z!kDaQ{3_ragK597&nM)Xn;#4Lh47QtY?L3NTe!~bmg!=VO;QRgB=lYtG?mR5VqH*b z;NM&l1W3kBkAWuy`H-6xdfSFmIZ!`C9?HKy!-m;=b-~K;hd|3TiTp_rS>0lf=UK>^ zEG_3h!0W9*H#%mk9Ln4%g#TVOo(TStX{QWB6xBt1RB=}mE|9P)LZ<*9r!s~(1leb% z(rPdn;h=spERk;3!S=r?9M0GA>^sv4l_4c;%0lZIKYy`Sz`DIkFr*Ge)`Cv@9(Mm5 zg{H~;N2LJmfR-uQsRx_@;B2cxaS(qrq;DGvMqeO56hKz6S_Y(nFq56LxC!-}<$6Nj zDQLgv5QF6rsRbBw8FG~r?sH=Qs+uAuFO8ZTuP;xVDl{nz8!J4Y9`9=sO+pcM-nY?Q zutd=HhwE?FWfo%JJyk`N9Q0vw^iUV*>9x^a0jrSkMwTd0Ac7|RzL$`CRQ)JrVoNU^ z=7aKht_*XkTapT)`QSIqc*Fo0Gb2%h4bF_{!8-ZBNvpa7D5_k}4d=C>k(+JUVY9qW z%Kj-jeJsf3fFDBuMK;zlA!Z4sqwf3rmp4Wt;2%v?n3!QTHUu^qQ!|{XVgub&@RvcM zP$5AbO;SJ-)Oo^L=kX_)o$gpxE1p08Q*S&hE5|fpl(a%`koylA&Jgg)uU$%BcJGY5 zoX6a*(}%oGyv}2^!;C#_Tve>g`^#5qK&BJ(WQM!>!UYL!W&tCB2hBJ*d=>Soye~ls zWF71?!YD1DQ1Jugio!zQ!5vCW9j=2=;Wg$lk$=vADi~C4B>bQmc7}-l;1LD5jdjD| z6LXp^30g~WREYE)PUa-?==D$w?8}iP(*uo84)hg^@Wj=k+!?$=CN)78w%}J?f)Qm& z0LEY*uxAw=#eLenUw1daH zTb4RdynIlBLp<3^2QV+8C!R{d)+uA$M)Bi+6~$kkj}+7MVdcP95Xtw~PL*;l@JXiW z{am|zM6?DUemD{t4~9bl&_gQZ(qmx?N*Nz8(Uv6(Fz?lor^unqC7^W#NgoA_lsMGj zo>M+_r4-zSe~#~h38yz`$+HF7iP1rWU13QIWAO{Q77)$CX!LWpx%23WLUwR7_%yi8 zb9B+41r{i!J=Y|$T?u=Ly<%`2z(_M5DnP~qczl_Tia;`W45I5V0gx<5{YWt|)T&7! z5R;K>Qh;qJgk7+<*^g6d+*im->P+TBpkm`MZ%FQ&@^EMu;Ul%NJ98@=I#2LIm{%fD5VuRsdmuB1teE?foGEB1D2&`8n3pLrt(1 z=z29`{sT55r;fC~gZXIAtrYd>Ib|NivWufo;SFRCbCNMV;F5))-vdzyCji*AZevBO zXbtk2DKE@m$cKfC5}O)PjiDvM7w*KD@XI+&%S`9zJgE!x7B9z%;x5%G(hs8p1kZEN zLX;D@)!{$MW-Ky}A5;ZQP`p19l^)h=XKaAgBOv8CC#;gC9C=dLFrJ7Qh<(k))g{g zJNSc#$(}^5Dna2o%o>~a<#r?1z28BX5ZQ;&|25$kp{k$y4r^=F-IjJi!_ut7y{!w! z4%49M4rTfH6M^&LUS%lrc?r#gw0#KKedsROU zn?P==27+}@N`d_{D2b`;=Yy^5aK(ki;khZFfUOMYLH-@-v&7ef0F2|I=UjE9(ztpZp>1tOe}wvegSq6h6D7THl}q%Yhb zQXq&8i}1mMY$4+ntFHhzbWZw_qBao-@*+=A*F&DA^)rZq=UOi=oHn0=jaAg;efXf! zUlj|aH!45I@fR@qQlM2JNpWu#L2V`${)L#tuLj^4tj38qpvmbj=O3fQDhvDM?Ym&I zLS-v$79Zdw35o-qlXW4IM`AJD0wNf>NL}-9Kjh#0yE7eojD)_Jltv{8$RiW1q!L7F zR112Hcos|feuw1wHVeWLGK>jgx_ZPK7@gLR9tC+KIYWNj_B2lq^1=7m9oTud;hu$l z&F#m+qb1oKm`EpYUu?RWAg%m~X(L*;b|qcDsOB}3Gx3#F6&sM@sBlTd-GuNux#pDdPFu7ppF$C)2uv$}o3qyOE|Hx`aYGNN(pY zm#4g$mZF8(D*G4MARxINKK%oWPjH9jj10B?ZRHA*SVgA!auTZZCV#hJ%1xxeePilD z^q&IFyuog#i{#gGtr+0UqHzRVK{tJ@W;Kh6^btMj7lAljgjNfXqv^L*)knHWjL3XY z-wfz7C^8F9G7+e>3KJ{Vh?nZqPKlbDFSlSoJg2C2| z|3iUFK`Msa>oY1i201{yWp0b8CadP2zimhw&nOar$~`+NZS3tCdg1)z*Wq8bW%`wO zK!q+M>ZtYnYA}?L`pSr9`3qiu-#<&qrD;y1p)_S+V!%CD0Ee62sW0r6qbOHlr+BFh z1sFC%&H(W=K>bD{sk?v02&DSOx*{S4=t1~sl-F4gD2F)QHt9O{5z2?ZEaT;pPma`J zq862%UE&Xz3lAFWJKD1dNGim$g!W#{SHfY58R{Tr4`({ZUtJ{*Q~3KkPI)i*KK!tR zXb0>yC-_{O_yz_u;hV~#{wwsgaUKPxkU=U-@LSzSqYOp3WHO3=*qwRa&`>9$B9JKz z<$98rFBS)bYucTX2y>|DZxHuU%SBrx(l}x%pWI?SzpVC5>d2=^sAT(^+j{k zt2w_$aITOOp#7Tk5aEunmyJjb*ui6o`3trk5eDV0eNe+1A-|>Np!w5Nf)B$0?uqbG z#A$3dme-8Aa}ok=7FAj4b9_~--!u5LGHVkVxRQ&D*e}M6(yN#!ktG=1x+0J41T$@! zE!P|%cdlnyn7gU_;TRJ+$cSCcOJmui!7Ut>Fcp<1CUW?ul z=jy!6;%U^^aL>z67G#3qy|cR(@sFU158Z=11~DR+PC=)YWCSD_x$4&1wR{OdDLF_{ z{_Bz@lE~qUDrlk2R(b@?WZC8ANb1;`byiB;c(WirAIjsf@m0<36E%6H;M81(w7-z6 zW%zIr4iyp-<2H&pxw)@Wv1WZdkb8q#1e!dahyM$ig0H-P{ z`9l7Swwmaic@g3evAdoEHQpUSq!*?)Z2HMXaVFDkP2j1jyJV#^`DZ24zghteA~eT=18q&=e|8?E6AbnylDi0 zh~|iY)Lr5(3@C)juBqe>j|V*?`J5AjQs@MwGa@=Zc3wfh3&6iucgts9%-MzehFi26 z$?K-;L!{$CBmhzv1^rM;3J9;a?>E)LQwAbDFNshQBSt{k#@~nJxAX!O^@*~^#=@EA#TvcMC22Syfj~8#rM2db=Hb3q@1~NhWZ~zhyR&^ zWa6_wi?7a3{{^Lq3dU}~3MY0d|$Ny~JSs?F#(t%c>iEnOKXg{~u13i{TKz#zt ziKknx_x=eM-820Z*Ilq{^I9Vt{(}V&a-gv1dq%PN5W#u(h zJ1_cTyH~MJJ)U7eA26<}sHeCpFnobHT9M71g+X9}wbdBovY_>1`8}V;V7`rE0oEaW zV=1sZ4fim221I^cl($b0#VSVML-8+TrQrd-Pp?6cm>}kF`lO9en2cbqU&WM&jW+GZ zk=lM%$8;Qbw-?ky1i^QcEsCMrUBYql0n<5ke~b_&ckOm;Mj}MzvA7QB6?3E_U`1pz9cX$?w-2`(ub6)ZR5`=#QxybKGn1FME$~j%$0rhaX-T z|F?H0-tA?HbD}aUN zzYV~`!N~gm31DGiV)=gru>7yS|2Kezm5t;71hD8w7_W36zt8n5k*U+_ZnqWL7mHqo{)KpdFjRVp8x`sZrQff(eOLwc8e^r;< z(%M>mNnPM``TAMAS3iuJ;AH_9<wJnR8rdjyaC`)Bm)@=5&0NI zkC{C=S!1?U&>`*0GfM?MPP6J%FDXuJ;_F-E@A|SHtOuk`Af%0I)4odv@>E=|Iv$Er z66}N=a_hgu(#aTYid~AKp`b~d(vUp(akdMg%c|C!1}jek191N(fF%cW@P6pzAshf- ztjWMl4cyt+Zx6X2K4?^ba?vJaeGaV6!0CuKARNWfxjwGA4}>qY)w3lWK35Xqo5qxOWqAZc;R-7wh0cri|)OR=sX=V00ly!8R95{RP* z5NHOxMrCGn4rmcKHd<>=$db&Cz0~@lLT;RIP4`Duj{bk7CGw3iS3$EE>U0GEaXKVd zg~G`+&pHNZ1H;Vx?2MtLk3m!>UZ@R! zt5$zShziXTL%uB9fYquKlO&x(Yo0##F?037AK>#8^=DdQEsI9=>7Q1_k>GSypib?Q z7!6hXD|dsy!K&Ct!4ViF#oZ0lT;p9&58*vW55rl$|#k2WqSPa`0 z$THi6*s~+A#ts@xN1uXWSBC;y+fv zG#yJ(^pEDxoOYEPNIJ9bY8bArV1HUTt7LmAcM-lHUcWZo2rBGqk^Yzo^AKFH5_(h} z;MPwsBT#`NnV4J(1=drOS1Bl6gH3xnx&m<%H{OYn1_43s9&&L&4>M=hcEv46 z3Q8%c557sk!35027}@N2Gr?7mFB9s|FctyMzIJtAZZSvJq>ug@OgC z0YXDRwt#-I))ss?mROv0eRYB8g(&b|ym5KGHCPjRDDOtyDA%S;=AvUEMP0-Nro=w; zK+}>mbr+HmHn;3O1SweNN^2omrbKLEKR*P$7HkzECW%Hv9mN=Ykk(Mmeq-P%B!h~* zwL;|_kI@=f2g5+DP|(nFj5OH&C5qZbw<5K}ZvQN7|pq?TnTLiTZrI!yf5f2M8EI1z*u^9Bj9AL0s zVq34Jcps=Ykp-A^MNFxdp ze224Slt^PsJUjku1T9S-sx~60`ow6)P5yaGsy3T8V$Mxj8(HqkI{r+Se_+`Utf19MHNSp$&tJ)slm#4E!!T7sswc#Htk{=WhNR z;T&zxPa+>3p2Nk8J~E%+#G{*I=~~A$$g9~?$>(6ysr z*`B#@O}h6>&%*(`Uhat~YCSEWMAh=WsmI2UxGkb{aHL3yJh8(c{I9-rQ40DVbY0NJJKfchRi9 zjda$o4LB7?cGlIsyQ&!k$zo3Kx0K}S9Z`2M6+-J2^pdo1bLv3ahygckrfW{x@jFq; z8oLOToc5U8Z>K~yD?{%_rEPf8iDy1CFT04jYU!Ne?Bya+y@u*v{PRX!sF4BWy4>?< z*?}r3A;RFHeW!}Wh|?5@1aiBeN?83gIILQJHYVn34lh$L2hI{k;X{?Hcbvsm#6lzG zlEm7j2h~BhRy?)JVP|%bzKuHD%d=<<5{Je~w(!NXQsd>IQHZ%BG&{bzDFyd%!~0x?aO?zkAKIRX!0PuysN|P_!{{>A%vheZ7epul@i)Yg zrv4$R_f}dQ$e+7IBbp?}m@rO&(x5gm5#YPI;Y&#pOYN9O*osqQh1z%_2vHeWq;u$g zxlrOn53!=jq+*MK7%2~H@xG^zmt6-Lak?w(2L}9ZDTkaEvxmoAN)l?5ipp1`=5aj) zm1^5$r|n1~NCgT-nRS&ELLSD%z%hhjR%DMe^Q0?XvX~Neng{9-OTE)mN!W)_5|rSG zk#U2~zj9I0*(BIG-{%mbDL_-yK?qr<;%Z*KtLCeuOT#WTWM$j18yN}K2DZaRhCp!9 z37HM*bKRf-O{ajG6lphUEPK6cp#%@&w69Je>-IKVRi$dn`pIM|6Rdca8e?jZgz@Ku4!KXa zDstJA{t*?qag`G~qS-db2zAMlvlZ&=QvFGB3b(~{ulS{bstrMUoi2gFkw&u>Cz3`U z=2DB48MR~+Q_KJrs(j?onL@FZ;eJ(D5O~`H^!gPfPW1BDqGa(<0VPrCM*6?<;_Zs5 zf}Q>UqVLHToIJAKTj+>-Z}bFEVxxv}X~=;C@Az;SYit;&SB;Veq$#^$1Nmf8v{@GpLg8bo z61ku#%l-aFXg>eEY_YYnl&mjC>?+T(iApRN$(V;Cl#KtU=h{gi{tp8JIVA^L5-Jpp zUco8wko_XCPOgr8C<%lg85aQ~XhS|z&?Q;1CPYng~vYc?(y|VQ}YQfnH+|*wPhd$r$tq2V`PN{rha3(Y zV+jsSs&$@MAFv|&Q*wyTZmDW4wc5vT(wQIYrRa6!MxL-;4yP&At@H7Ao()=;moK>( z{bSy7`(AZzW@TLsJvL$GK%y7b0 z7M%@wHd?tDxkFq>w^QUhd%=i-B?6(ez#p`jTf_O;M+udX%C%k zSs6XxLi)iwiqPpzdz!sBX66NoGzl()vhKd~U%nA!`q;Tl)$acYgdp8Oi)n%6#^+}c zasm`E$DX1?DU|6Xq-1zrB^+?gdK3FWhu*(E_|maI4=&PeJ&C#E8MPKq4WowJw|n(Y z=jFl!_n9L`pC8MrnqSvjC<6W@086lKh%<&rwrJFya>6pw?uxI35PJWa+ivIfMsr{8 zbPk7BKdBL^(UwhN>L2N@>nr)8rHj!hduNWZ*96wyt*5`S3F2CNMdeHp2y}Qdkj#&K zdlCW7G!Gjkr;IM@W4O%Zb9I`q-}Y|wSC~B5t_y^M<*f}kem^AJ4wPOw1-o(t?cTSQ z5GpewT-d^ydlw{+tuF!#x1b5_-0sYs>RxOzO~sfVgv`xSpf2MVISKKa7JKtcwmby` z8i^f?P@3)_h?fH^TuuiHMp>Kp?|xfNxAXksN*BdLdOnSOz*}_q`&(@s_@q6O8h7cu3Md%EEvE@nSt%QlfUt59>~ba6nCGMFW)`&AbC zn3a(uF>pLTGZ)aVjF7#rbZ%f`jLmoZe7TIMJ#(8ib zh!-%AKO}Cu_wIOIabP0034dl+gUq!Qu&ER^Our)fnMX>Y-?e75o|~GU>+DP%;1W00 z8Qi!J)`;Bj5;m*n)rfmfA2PL73)jTx{lbUJ{A39dhtw=kdsIuXxN*NUoF+Y@3FcF( z71^3y`%{g2YvGG{D9&$9yFSdMoW!+dYS@6I%|ETXx6FN{SRzW3Sr&MGh52UQ<}wjI z?y0ql9p6TMve<^dB{h#OJKqjBXoKp$kFX;Ka~bAH%ZBdmC<@yL52~SEW($8~ddv%+ zW7oOxcjyxDl-ttHZ~8e+L&y1aUQH6+sY^h=aK~;QquM#ofID;UNv^XvrwTl`fPOQL zS4{1mhi_lU#mXA8DYWnTmTUn#Ur*<}C>S@jmS-VGYJG8-{D2OGKbW7FoH*kT;NFZ~ z1JC*2Xq@+AGW~<(;Z2h%j$M1((^hPJvSIWBR?{If6YcKEJJHjv&o6`{2F7q_+ht|L z9Z^ba&Jdi!*GOc^307=Q)gOG1J>-VO3o|q6E~-{FgZM!_gKv~^{6j(r0v2M>D3mw9 zo=6{wRN=ZDIzRxcGUng;Sh)mzui|OL6Gbrnh^EXTN40nrZDm z$pFK|e5L0pE6&k zZ;i+4UF6J7<9HdL69_6qKuR=dh2B*nbiC7EpCGOWP>!?(S~cu+dd)~ zdwROm*zl|Iu8QVLu(dXes#keL5VFb4C-#piEL^gQtT8U5lQO1LmWH~q&-~SK9b@4e_qTWblUI}drs;n*Ao4TJIy)Sb=aatvL|s0mT2X+kF7HhITB9G0AECG ziswB>S=f5Ek|9cLE5o*Ru;7?G1AbHiVbF^ly#{%Y1Grv@J{7g>9Cw4F34aJdSx1>m zkF+Mtj0pBQ@io|0bwpP_Px(nW7>#H|qw2Mi%W$&Ku4Q9SNWXj8W4b!yeU-oWfWm??t43#}+x+JK z^5sN|`uFQ??9e1lBqr=JX(nL4ci1N3DT{W<^;@An3QnL-r`EZszmNu+JJ)Hcl2P2u1`HHf$+;la(|hCQZM(}@p%VBPomCn`j*9$R;shNQf*d;S>n>xDvj}^ z6cCBd>AuucV^Lno=E!Wtt7mKG$XCN~{n8SP1!DY8ZnAi@3l^gLs0FGaTTO z0~@{Pc6B67+DsMIiYyy$FYJyoq;Ee1@_f@|Dw2|0gsj<`st(^IpF=tQtx8T`F?d(H1KrYQoJO6Be!9Z`9=n zq&spXbtqUVPvri5(K~Sc#pzIWg}*?k&z?hG zyM5bxa2A;2*p+sRv?5jQ<>PFQr>kU;yFR-PHG@cs0L^2|Lqfqn8bcZ(*)hk&jLc?y zeksg*phTSSD0h7ft937UUgA$47q$cag#JQYk}(vp^!%g5S9nh&>Rk@@ZS=mb4g+Nt zMQ`ZoRd*i`K{b~${HliGz&l=PvR1)Cxwj2DHg^k`w>@!eD!nto;oq-H*=|BhbnUn^ zb`!E4_?2(a1^2W$Ap*@vlXZh}tSRus!%gvQGGKyKixqT&z7RJzXchtbH#np=bs~q? zh+6Ny^_>Z?oeR=QRk1r|#Zqd*c*8HJ@XaOoXGDbQ`;C#2$UTfZ_&BmIWRX$CR~Ag& zOORcAfJB+3;g- zLXH*+WaeI`6e7a=SFWuwCM$4$k@$**=8QHFg|C`LvuoQ?XBHO>?{tl3SAb|3=q_LD0L?*mnPpv{I)h|X=e4`o)z-@_m zb*YM7RtV2#4^4lmmZ~j$gAqY4PaFTNRpwAyZ6y7;gvSr=+gm)M1ytu8tCDGi%x+|= zmE~sM;q^(?|B$ymjVqEzX)xdhJUQMSve`+?AfA1zPbuL zl_D=u)MB-FI>{Z+riC5%HG|yF<=$!Xt0q*U1jOp-GN(zFhaZsxnpLjV3Cl=RJR}Mp ztAOphZ#K$B%UkiFgCaFgfuUr> zUVO&M2m5uEdZcX(>oJPt@QD`g#=_-EP6`nluGY9BLFZJ8yYH!jkoS~RXZweBDv|oV z9)nHKrO)d9iS**VMtws<^nF9?#R^ao8~xA7Nz@$FtBf8dPnr>U=f=Eu{Y`osjzuPm zV0qcx1{D0t;u$Wmxn@K@J)DCMcdJXns$2a@!B-z z=)F{0H(9Bu9jdQxx|W?MqnmTO+h_<+`lqyoxc#Sctx}{wR@JydIHE7U#0pDjS+H0l ztej2nK7hVSurByMms7Bkfs=`UTMW$r>kQool|c+iPW(P;(c9B#IhtO?49C7&Nr!ck zeBbH0KOU!T`XHL0O4X=)@){ZcUilW;h@Oy0@78nu_lD<}U$ox@nl{*ehcn^6$rYTN z`ZRUDLONpJe zoy6gD!&9VYZg;VQ8`w=ZtYZT?M48W>z8dh;(gG~F^f{H-WoU)~l2LiScv^kIEz8mTaBjG(s#MIw6%Ldgwxx>IpTfe@#V{XXNZu(KDrJObUd4NUungGSg9b{_9hQMCOEnd-Hy|tRUx| zr-8VG`&gncHg6oVbStdIIvq|gHSm;a^&idzOwTOjGGA@ z`+$7HV8$~}W!n^QkikO8aMBq1QKaofUE_?lCc8yNa@qDfd>72b?7Y_ytcz$(_rdTo zbQ{&P#d(V!{O6Yphp0>~_Qp?cw2d;E?{#i!7ieuMDU#zR3Kb7po@yi;4Rm&N75njT zbfAIW*dSe(3nsQjZ11AF9BLsUldMna*WEsPxtlH32Ym^i5xU>s)`o&=^g#DtJTco@ z6vhu8vI=|>fS!g{8U$0~5dCC>QgxX86{MEd$CfwHba9Fwow_;(ft=Q+_B~JRbPjv4 zjyn&&E4q+u2H<6hU;B0(=do<4zC$Uuub81Nah!QcVt;&5gl8YTJx;lXNoX>fJ2<3C zdLr6@p-6R&fc5_LM8K@B&R5j@v6wxXWfrkP)I0*fpvoXXvZX@bg(%Yt$N+c_&X5C> zWln>G8wR?!UdwIvRJGGnbo3K^!}mYcOB(7hiluF;I$c~feSf=N+_evKr$kJmGmSW| zwJc%!UhZ>@{dn)eN}Q)^h-HOD(XHxeY)b^=k!r9F`%XP;R`NZ_+^`Szimaz zh!K2_Xq`{(i|<7Q4T#B@GQIowSqtlVh#Pd_3D+gfff85Z__ocavvIlbMy^V}CUJW> z-&Nu7*X~`2YMEN9rXItO(8^KZyL6WbgcuxZ0i#jy4HE@l)@Fe6t7jHTbF1NM`zqvMWJ{4O&foBd7NIpS(uaBNJT!?(HYi;=$Hlaj~&X&r2myOK( zsnMZ}F55%XI!F)!y4xg=*m~KU9D!e$5#^n@bC>TtM`66j8!u<*w87f=6W2S2IADTt zP$ve$N`%*-&>i_vH8GUsnWUQb%Z|h$gtl)`LUZ4rq?n*H)ZYWAZ?cM%^C#meAC>z>h-l+j5;zLQ<>Zy4~si^su!R5skGXCvnem zXgYfk(*fgliRAKBW>T=IGl=u4tHeF@AM{or-n9i=QTWzDK4}>cV{#>LZ5uAiU`CXm z4@gfQGVqJ)=tHSICWa`RPe6czf)YGX+6({`x8tF*GOa$*)%6lZdK)Y_4OOpi-IH2T z@U(TwNA$^ENvNb`dWTng{vvbL2T5-4Vgiths`U ziK8Ciy1~@ms=Vg7&)WF>^~4i7r=ZNXU4+mSLa$bm#DyS$*kbjb$Y-0^BRmah=JnGs zK_zWEPcM}+2Yld}9Qlv8xJUVEat7}=h`eFrJ&vcb3OpiVERo6{>@$JPS5=xTFtAXa zh1$o{1CndEFv0`Iz_`PqJ`%2E(2lH8RWhOo8k|~RtyA3)lu_jw_pUu6`PRi}j%&n6 z94D`9m(xm7t8f8QeJ-qKVX&kc+fSQPjy^SOIDuYc-y2N?Utdw)8|;9>S`ZraCLzSTBEE7f;288A|Ne(1tn8eWs_pzt;#8WQ3X(AN%0;vH+g^S#Qqofi$u z5haR3CtGl^>s%=vm}~>#YH(NgZ0M_&@E1DfMV{%n&(OB>$4mC~trbMyUwFTA2z!LO zC-Y3b`Y4YsuLd%ll5H$g(guXMHf~oLpN}4HXLZ+%Jb4$MOySA3KhZH&S9S9t*MbtX z+1+l=?XE;Rbvr=Ym$4l@szK6UkyYBaY^LZ*RQf$0w2k7-g|{+% z_o!#pXlJ!1$>1>H4uWRQ2WCczSG!_db(&}4FXBGbdAUEr5p5Z6mg7cz16naY_93}K znv$Acm_8;}R;{zd4X9p!fm{e<*|E~m)jp(YQrPiLBcF4{0mfi{Sn0}n7@{iJG-&Kp zeA$Nnrp^~)ccmN1=!e`_yqEsbmGP@^`bs%8JG_K4RS!_Cuo<=YR6ulv0tGua&65{!U z0aM!qwQPfAm0-x4{C?8MmFAJ(-!qf_8T|R|@DYo+VNi8akrrhT+pQPFXg+kJZBS;} zErufeB=tw2f{_3t%Sl2tl#nDRh1N{BMTC8vK*h@vJ5h4`VhmB(1r?lW;VGO{oM5*q z?0y-7lCFU9269nMq$kFGvtz5NTNPn?K~a;Xj_5+^McAaBgCPq=>@PsG8%D?}R4JDH zx*aPyX5!2Z>@sBSY?delFynYN1;P2%3`0ZLf>&WwS_jqksBj9c$Z`nH9o|vaG%Ltk zNXt-T)V7Tz?fSW2P@;!qcynQ7oaLYT;T~US!}v*K-AWaW!9N*HH(t3EbcQPY{;d@? zBjy(GY^Y`GEyu{@{M#91PgeAvTZjeRl?N<}D{m|c?|cc~`zcpWh(~U=@$xE;K{{_+ zG=^t*51-eUPPReZ>)sW9+^zY#`6WsOTlFF1DV#gvCxK58%d45?&**Yd8BUf`{pn^rbd_WkR`R`p zu#bUlFVdCyTHM!G;cXj1JtFsz<{ZIcLzMbvqCQ{2TgYfQtJiiN!kW>SkS~E&2qAlQ<HUbji3?FtAZA|EC8;jo_Ii> z^|?vfjJ|D&q6%egfbuqlYDOzT(_6@7Tp;KeguI6CPrSM~Q8k9W8+N@#s~M%anM(3P z2fcHHcHFIQTa4t-mqB~cL)}k`nU(0yOw%H0%G)%<#%_G2%(GS?9E>64fm+qjb)9<> zt%QgsB)pTR(!M<~TXXdtG3??@p%z_<9tuUK@uqDN!1@(Vrg!$Fl6<6^vAuLFI|6e^ zYdxI)15uVS)%_ZiIQ|@l%F5t=?S`I`NvvNYnF8MfY>Dq!n5j~)6OpFuSr1F%Zrv)~ zF~ozE-wEY=y(D;vV?&b-I#U%ZdaFIYfy>N$ndmv3cVfM*5VKAQOGYlJk1Oyldr;PKLSE&?=Hg3=73o5~X$&w6NUVz;*~lm#{$ z&dF&YoPEwgnG&#>hx~89mvqf%5*8R=%jSWjSx@sArfuBOEZ3) z$R9m4z=po>D%{N~r3%fVFVs6Iw%ae!9bym~6XDQ7kY`=gGrj<3GeG_W(&CcTj2^5f zW72CCWFn24_yPJtI9BwB#hTfa%38Z)9(RK`bZq*&!chs!Q?L3$zd{@L$}%DNzLhX5 z3~F`<03|HjeP6qN?KpUbN*q+bKfF79`7FmY*4g*v&0RD6(aqIJVH(vgKK*v!j(!aT zXNveo!Nz@H0pvIoD+FvRu#^=7NtFOrq1TbpKG~_?v;u@nZZo8EnRCrk$+lFr)yq%E z+>|BQ-HX(?8JQ}^OxOk4{&(Lpa(0sIFHD~n(OZ}Zzl2v__KD_%tT%W^?mV(iU*}#4 z)tYt{_UeNt7HrfC{&BA&i zYp<`YUZhTCJE^oM`c2(v=VxncRuEd*JYss1`M)92ESk*c;_jZJA5J*MlZ>sssc?yJrbEIAeVVh(?g=ZmPkQ+2%~s!5aiF>OojHBv z!Ns&CN>|@m{#!L9Qq~dfmArD9QL9)#lCHu+QxuyFjXfh+OLm!{(*RxBBQNBocsjTe zh$6W|T*CB0%jk>@jt7_*uWK1$DTty;<65ACZIgy%?5UA5%d4LH>POjQyPYy?Kx#%F zLz^KvHfMyzI>fp&;n*2rCbjaBdmMWm58qO}q?MMj5JM(KWzkgE=STrC0+vbvS3WEB z+Ji}j7birYpgysP=HXo9H6tgjl%|kH8%*~_D)nd%?6O*aMl>`3I02yK@Jq@Ea6Idk zao=e=F1N{pKO_slJ$4w0D_(uo{~0B+@P#O(OTJxj@~i5^8=<+tnw8ua22n&f7ZBcU zBgBe$)aU|Qiv`j=5N^bXXX4@xYLJ?8E86iPSFGz&oJo=R8t*N>=Vw?0ktk{G;8{iL zP1+tKu?;M5K0)hl1>4ab5*5$Og+%h?v4yA>R6>0s>1wuJ@~<0V!BduAU3&W#=jQ8+ znFP}{htMV#jS4~~$icw7pi}(qH5?@yO6_1L|ykN?f3rc=s+b=;0ubdLwxyWp$ESu#0+kHT4GqZL=5`Z#Bew?P2EXIH_sM=q z{J|i~b%=TqNjx<#bsV(gA&qY~8VVy~;rIUYRa)h#v*6q1TPN7Jl+BN51?Y0MpoWxY zcRIv7UKf@+ythZh6D|KY)O490 zP6!?lKJxgDC(N17BcP@bB54g;V{8)W`ReY)r~USoknadu_9pa5 zX@7w&m-Q<%EuPZ#BXe%jG%0e6W0$=*_vpI^PkCMRr~I1iLod^CD~u7TNb~62?-6L~ zJ2>m;+Ji5p#@?JONMSt}A)TWWP(ek+2;8#;tZY8N((QGWS+z06ep z5K3n$i|pl50W8IL7IpVCS6qNu6g6PseC~Z?I)LWIxB^|+6~XqZ0<_9^0NVo#{x~u7 z1noq~jf8sn0s_HS=Gm~aFJo*y;v(jIYs@RMR`fauNGsa;)N$=;(>kb60&owX;}|+v zu}=>^hu@B$Hb4gwvC3dp4lwZGEK{dC8H7PVHa^`hwK%EVbB=XNj21`H2CFrZPz+cZE0o0094xd zFsO?8sAI2G8LD}%6WrEnb(Pvi{Cm6U4Ph<$$d@3kJB$8B=z)E4Zri$k*}xbV=jSr1EGORdxEJrQ+Pt-RGm=8X2Y7=^QQN(pTIwR z8!-~X29>M>YugjHi9npO;hDJk@)^QGzJ@t4c@9y zJe!M6!j97_8v_kP<`&&+ot^KhmvuJOiZ4F!6r;7Z6zsuRdkj7IFMM6L?I(;QRIDTD z>h^x|USdn0*0;-h=fS-V#^;K8A{MG;lgoOcsj^yb_WD5hUinZ^aqsit(!M=@2kP{9 zp|AU(i`=`)GWgmiOSdamC8;mAHjgEm*E~vQANC+M7S|*yoI$TgOMECaP#sAN91Q7g z12n!hQ}5l`zJJ3damg)IwkTec+iA0;b<1oQ@aDR+BXjYFa-OmRvbcS0wO8=0*kRRa z1@{<&+*6Mf6tdy?IRRv5r_+ki-3kVp-z2+F`~n6?bQl_QI|PHMKKaXNpwGs_xs9NQ z{M{ymik!Qr<0*83;6uN>)}5=hnDu+URPV6}J3N9YK88(>DUV0^nL)z04GZVIy&H*w zJKH@iwW5r1&=Sd(yM9tzLy1WO4Bmoqq$c(!xZ%}?bS+AWE$OK5F5d*KO2^5VR6TES zvGQ?J7&JcHBIUm`xbX8(VA!f6?44Bv&2+cbC!~5@2dqBua3r4#c2Oq^84V9omV_tY zq23vx>jcFk`aG3OOTbpwAla6|qlhEjHhow-?7}h{7$r$GRxcyd$}uzIVlG2b?KLX9 z$L#Mr`pTl-A8qQ?t?{_q@n)l+!>_gzqD+a%wE7ubKOUpl6-QO_PP;LmA0tf(_L2%l6!3EIBaV{xHEEYNjU+5!{i`w7q(`{A!WK-%LkM= z%yw7arK5-f(G<@{4}gA6Cf4MYEw6rB!t{wd8QB84=kzi;WY>wv&9{L>l;AEQ!B!Ia zH3wAqaC)!CTl8BG!S7@w;aB#YYpeJ09s(4+-&}%qIgA9hiaSg_pYHpx6SAj-OROUr zk_p^#TucUs>$^T45d^pug!@fA@)JcQ$>a;K7@Y>+-vQyY@5$S30lXv; z88JMUp1^yEWqGfVd;J6QM&YCe`qd$OG`+jmd}q;l@B`O;^#oZ0;W7f=#Y+?ccZ~_4UZqS+A7<6!h z4nEMq2|Bny2lyWNIp{YB0F;*Z=Y4k2ne*2|x&J9tfa1v?$>I2K*HJA(|5EyQ4NhkWIzVS0 zj$hf~{fDUku=5jIPPYHb3Ew}NesTgX6;vCD(4RGc&R}ps9Q*?=4;W}Lm>?>CPzmN5 zoDW3lKh%NK@Bn|IjvMe#MgO7hKeFcpN&mmeo}2xboFWwLoLt}=N)%7zj9nc;(dYQU zEPyT*^zDtU9Vz&Dz`6pywl{VLKL*1Bx_o8sXlV@24GxT_^qY2pqRas(z=|gDFA%uE zVgiEzVjRri4{ZZ84AQJ041>Y>N8CSw_(vrFfWiqb11u`AHvSOR&pcr8z<~bL-XG;0Mb-_OAHfa?2^4*U!(2(a4xQV-DNS7H46ZT0`5 z`CwrH|BL2x|GNqQgbGX|xJodI;Oaov|G@S`d%=416RID2^D`~SPu<}J!vkg!1j`Th zz?%IN8Zb-$lnvDX2|XCTpREC$o(o(rIQ`ES0lxR+8brsB^1v|sQy!RBFpc0cz>xi| zp#ViM{HY)R(cPadi|fC(Ebf0>m;b+Q#J`aE!TB!={6k`38UD`1u@k`nQVzkSCZ*u&h9Ne#rEvj)1M{ z56cR^2J6`Wkn2Bs$O*0+oc~u{{Z0PAa{OOcY7Wof|UaRvT__; z6s$nd4-O6vZXS?*;sUU7@Bu0K*a2+pT-+SI6x#pF1d69=%mKDn;I;~~U*Lk|^sS6R zRQ>p4mN7Rlw$}%Doyz*w4irzM9Q7^D4MnU?L1qpB>P8)ntyCyDx!8F5csYT5AOq-T z3(5>G228NMxh*(&=dYbC&#xx>|6|tKc{$ktydVI6XP%pb4ZzM0h5-md1OR0JKez|m zo8Pz>{S)`BqBfRB{{;dbZZ>uvc3vLdzk%S_zUGe+O@D6fz`?->#?Haf-dNuX$u*5j zklZm8QNt&%>@(9Z1b7y-?ve4KOhA82!MNIZ`C1w9#`B^Esp#&u%5}97}G? zM0dvtjjwxEHIWgA|12+nczxP5IW5ewY~%ghad3|2Y7Ng!V{a+BZU857V?@-KB-gEx z;G$@&IJCIONBprHog-mQT}8YNWn(eI03ziT)O>|Gij|0abD0m)hgl3A5G{!Q9&DWY8yRtS^YlVmK&E7)00|JL7LN zh}Uc}9phF8Gd$3Iy13eq{g@2JAwCxqIQQF7mV(+9`RG??=j2-EPIj z69cvT;rJI)Lle9Sc8kcK#rh5^7N0C{kww(*{}aM zl!c3+`2rANj+zZJ$0wn@qS?ugA5|eE+-K9v`oB@e4l;Sag2p8;QuUk?Vcz2% zi3klvMuE?RqyAAF>$kmLF+(WC*N#`ihK*g+$Mhpyr>hRp9F zlLr85kTdJ2keD84=JMeb5xCxD6SuZV-Rq&7sruu^hqmJyUa>#&`0q~t|AIb{m*;mL zW2Gat*%8FHU2rWaVz1}-+!Wu7R6UxLdp5!25$tC(6k4boNP3ep&MTP?*?9H7UU)3u zmbPd(kC914xE2`jl5Q4W^cvo-?6<2?e)(CcIdn9lj_o5~ZOwKmR<-fz(S-HHOD6lY z!jsRI$M&IRSo4}$>+SkN^38G>Dz~3ZWew23Q5v-8icO_2Kaql~N8rd)P_1nB!LuPc z@3d}IcOR?ilGP4h$CwPbxXw99j4%n2)Sk=WB3yFAR2O?D$m*S+h)S%nlU*Z)9#^DD z854q@Q)+0BI?Rhu6EI#V%JiLeAq3IseHrt>8}qK|3xfySN!R4K0VwjoKxymj#(uf@ z7GkH0U`W{LbI7j0Xsp>C$n#yJ{dOGCpTLqx8_h7P93#A zN8G3_v7sy(sCZEZBw;Bg{OB^butMe$DsPJN)q_)30$?aT8cwOWO{y?jxS~`o2q45Q zfEx6-GIv>wVDoXlKfCjK@PsT8h?V}MG=KGe{x^-~Was@&nu4)X_B{Xu*h?W;eB1aW z&a~|UCH1=npHw*dSv3MgGcqGNLIcS|YCi4BI?r!;eS>FS=HVQUD2VEG)~Ah~#?Iq> z7hiaJ*)8k4al+nP88F)xTc7A>zX{pH%JE(I(9Vdr?!T=xxL4P_mJY4#P7hacTX+RU zK<<<=N+!ShHL6b^W~85N#M}@|P;3=t?Of(0ej(Q-`I4)4*?N$bj9^o)gArd!f67eO zGD-;YLswOri@E|@e_3i)&kK$NCYPIY2^ML^n|n1lJg7ITe$FXo6ht?B@5K$0R5pjr zs3CR60^55d6y1&e=@GF$6dEpS0w!y|=*Blee_Q94;%spnGsYCH4! zn!dkDQ7qTt?QUKc;_g*oGd<9h5G5)$!JxWs$0bN-MpeExVeOXV8|!Z6mu|CBj4l$* zWRfX~n>i+>9kwT-7KtlcaG(v~v}@(J`L@3ivy>CBMOPR&i5>YO(Y$p~%@eZS-hG7S ze{sB5xK>6pX~ZtI<|yTxe?QtX_zF09YrX3eQ{2K+>iqyAk(*xpM`8clYyH18T@H@l zn=Y#&zqJ@U!oJtLhW3|RZ#*;b#TjF7>-Z}eStJm)0;sWwolhDO1QD1$2PT`E&gx#l zh2WDL@hhE_EOgghZiwN_>Zi8zO6|v{A-KZbt{LG6ybJf&Ngs?j78#vY2=eH(ad5*F zKia7XBC_l*Wgzs^rSgmKM8n-FZ7ORrHS1y3x9{H>Z&W6y`zWHcL9MJhGQ}3!9l-YF z9dnIh%LECGMC!TK#6a&YW!Oz{5Ml(8(m}-&M*5u!V5kjDIc=)WM_cQ@9HVikQ2n1TO!e=flxY@%rhZW;nW8QZe~LDrv1qf|4>i6y#dAu(l*# z3A)yP^;F{sB4vbk(P1r+iDbIXnpV(jFrS&BGI6Qq=2#k5t)GwX5~hCqikAM;W5%KK zA@HK%@O$ZuGBQi197|5LWFbt^!RZy+PF{^9QywS;gG(WAvk8dC3oaD3I?VW6A z)-60q2g3)kCiG7NMU)Lvb6L2!W?XXRX8euI)k4ocS&yJ|K2sE6Hcc zD<0~@^VfceDr48r14nPGB7OQP<|2`27i5MRzdYZ3?`mbfpRNaKMuVMB9y!L?(=w_ZfD;Z36~B0a%rLLY-1xdxEcSf7s@_JV8idscBw*CpXB z*r7;D71&u3yb~3(UrzowsHpQtr2f`V2ZCmUepNp%rGNS9+qgH+Akq`=Gf@3mV;}1% zH*?iO47F)$UrWF0++2U*Ht5%Cq! zhQqWT!1t_5u+Ppc)ER){GO@KVb989tpAQzXJ4!7Tb|qkY4ZE4r)t^P{41Tc70fj`I z5gqTB)4MzLmZ2VVBwnh7e%Tu!0w=qJ+5+kGfuj&|dGL^f+(aO-VDQmgXY3ZyY;7qs zEE?5S0lC#z@>Bo>6{3?A1n#dqPb4-WBfd5{d|i=1{-nE^Ekb@fZ&u*#nu|=l2*W1=95lw*;O)O7O2o4xs72UnM9W z&5s2hIe4TSL{XTFw7aDZ)Y~2|!-=TE6iK4-o-g>UWS?-$5iyGu7AG;Q-Xsv!C zoxhI$Aden zfl(=q3!kW+9xph)^bWGrp_Mr8u1x1AS1vCUcRlxLy2VILJAHtn?d7fiBZq%GM&JPc z-l51UHbj8Nl0(|C#A?sVS!aM@J#Y^91(I%T^)G73VyiglVai;sa&b*?m!1-(SlyB2 zTE3Lp7nKpAE5jsG2q?HHWIeJ)jWf%tWdBgL5;mKhgiJY3rudD+#3{Cs_E=nH6wBewF`jq{3yep zT5Xx(ESJIE_OeYyY5vMUD2aj**axF7WC9b_9`JS!aWd3ps+*|O?hW4?qb)cUMjLnh z5;-e_&D@X6=MfJs)2>d{H!-QQ`V6&27z68ScBq*C=J&Q4#6?hN&XYV&{Q~;$Q|%o4 zxI&_qtuL%Tmd<&3L)_l5yZ@1)za97hIe*_RQZQE9x)*>Dc*zHgA7Z5QiEm8zwRZ01 zvNN2dHajE=d2|uebJ0!i!tm(x*e5L|I?G4K+)uygv1(xV}Xs_tzP(1jq*Ak*|MPhHBp*JLLR3->QF1JGZ3QhL!F?(@I)ueGZE z5&4bQr2g9xCtIrW`Mrm9Hmllc%8CLtvnP@MDLO>Y`wDRjbI&-+NJp%sk>ES@* zcJx>n&0Ny3fn?~LS;l%JtQclE(-PR2h*dAe;hx)*-ba^5E#a2uns@Mp3rh7aeV-$V zbc{}5c+?h7$o+Ea- zynBG=kq&tGM*;oqs0}oa^t;AIOWVV+BRszp3W|h)vb=PA8b6rr@MvYk!Bs6%&N6J% z*i|9(;k46WbqKyR8s!d+X$u8_e@a!4wE5N}5cVB0CN}Qm`irBORoMn5LMF@51!cLQ z!mp*1&0N`$)EolT={B2Ox;UB_b(HbMt(r$2>9Q;%`9k4AufpuhFN`F@{N5sv^(ssIq6A7wfaU<}N@7kw0(yKf7kQ4;?8)QH#?4{ZP% zCfI*WLYt~XdUw;bpXvs&{devmJKyhy zkqgviLW9)QBh#Py7(UY!n2eKpTNfjx^wr&v9U+dY`+X?0r}EeQU>5wIE{|6(lY&Mq zc1a7dGVbRZb3(a_8%((>M)Q%>S&m9?Eg~$UpbZ>ZAHs&nOBi|Fm}sdmTuQ~Z7flB- z$R~QM_lX*Pa%l>Y&}cp=UeGYQ=`nHFd@A2{c}vikxb{78n;u@nk3uBk!&98L-p5gk zectgm4Fl1MIZ(r2Me)-B3g1em)FnpNcqtwZPQSeo6pnZ;`RR{f{Iv(j{<~3SRcx?= z0L|122H_#--G3LrhhdDppO0?Dfj?mhrP~>fWz9IXqrG^!NSz59wJCeUCpUGoPAG(O z@-dCQTQp!I>(FL4bW544$nrRwCKdE2Q8M~Tqf6LKWnQ=qhg1uT{-6O~E zmLvl+YS|=|NZl4YH4@8voN@aL(mRX#0HZClbmfNe0H}wLY@%YXO)z+6u_%iQFHs5Z zs$2DG0Y$EXBKRg0f~YUEE4V~fo}{{?8kngUbXqs}V@PMPekw|`IM8W?<0vTSxGaTKa4_boD{r2x7X4K^p^_R;&>!5;c1o-^kkMew&S z9O$LZZ)THAzA554SOl>6yw7~*&zhgxjvOq%t%`dWD5hvsB)UQ@Ow5P6gWUEsMT7PM zeQLm%g`tPh-(G+^HY~7wwvKJ-7zJpRt={r_ws57ML1j zDtI zk(Mhbg~zLmx~npqOQuHQJvmcyohgBvci?vmTZKkA_(#(I)~({;`h7A>T!GIXh7x1Q zEl3FP`W&&@R9h2m`cc00Eg`?fL@>>Js&D<>d0rF}lCFJ;HU25k!o&kfAqpY3B5+D4 z7DBLVw%A(8y~YWuz&%(;SuLp2Jv~kB4%ycqU|w3e*EZo_VbJsvUNO!jN39=B^`A%G zp^H(`dm4LDJD$Fno-0&iEO;rDXE*4wzDvDNJ&R~xqO5H*All4r(wTUgZO?wFy1D#D zbOYEwRL0_(;0%LXbHoyHg?Rd z*$qHrO28!`y`@V4cb)&G7syycY5HT;f9)*T2+< z8il40Rv-+T36E|sVMdCBca!f6UZm9$VZ*Tc;-{vsF~%FOEBYxxScEFWQmTxWWRP>H zaB$^K(PsH1(c)O+Lb^e(VyK2Wod6RyGd?-Xa#vTk$0scQ^%pL?r5dk46uOb(jLp*Q z5+5D$9NmiL8}&i9YS)&N?G@S1`3yaL2L@`DFSKR_~Q5n_jO+9 zl72AuMxnWdj`G9_R^)wi^G~C^ZgO&5W{++;CwJe=iUu3c?lF>2bLKxaBn_IuHr!ni zWEmD@N!ib~*_mZz8l%_&6d!J%NLjtVUW09qnoq`$NHC9qX`>08iSur&9^}UQCS))m zXhvU&iLp+~d>BDv#eu;Jmo)FWOl)%goDNgKbrqL+p_SGDh7mHPGwB&yCl{Vd7DtxW zko^n1qFleqs_&?8;et(Iz}yHmd$`u3i3vUvyAWoa=;AqbS_O5MErwcC<9CVVXlcT*w!V`f-aE%y-)G zz0)yr*Po$hdT-kIK56B>X3>bWHV!Wr}p zT1cjk+*L=hP}Bj>7^v7+d+Ehu7F`E<0T!#kW>|frz|Q%VT_b$5E|t!N_LusH01n5C z9Rf@rI>hvS0F6JXHd1nlGu2)GK$X_Lmjg}%b|*4TKuqvB_Q&vnksc8pBWDRNqG=J< zr6ec2&l8PJ{+fu9rGnmYm&?m!M<+K>#6TZl>eLN$VuaxzD?|2s(-7 zgvRQCPWy!s&L$Quj|B~k)?}0DT7`x5j@@VF21}&T8*BJd z7cu+29Q3ji>~>RF<*XtznNx;%dHVB4iRMI4%UpV)oru$hnU%CYbt|(}Cvy{|eMqE2 z|H2(Blde2vm~UmoxvfkQ4=alkJM=x2WBRkn6bPxF2$A{ToAoQ{awI zh7_---ips9q8AfzkA1T75xcLy*T$P_Z>Iu4>X*Z#X(j~8p}q|)rq%bH@+ zta$(GW7<)Va;8!9skdi?8wY3Pb+iswYX(V^I$cOAZUB{nQ4tb#7#}FeyljjiW}m%v z*h^`duvtu#UN+?Kc)=u1imj5>F2HcC=U!hWs?Rx(rQOAAs4hA@26WopBhqlGjd})X zQ=g2>k}==BN0#{HKqi+v;}pZjJ1=xH#ltz0y6#AUyG_E3$F8>hw1S9;s!$Z(VeXv2 z1d;G$O-HlDy{)n;rl_7i>;GZzE#Ruk*ZqG$q)U+Qln{iyH{Bsfw+IN*NOvO*lF}te zw6&j351K^#A>5HLqdJJM33iLSmuxdS@|NMFt@mP^!ri$F(X6Sfp57oK>Hu1}Go8F!m}}RD40Pb3!FQJg73DlshUPhlU9l znkS5xCU3c`XCzk3P$p!h+q1kpm>uHoRZwk8Q#yT6h_5(2hm7)OUEtYhp`-N>3j3Un z%o*uOP587a{K_XadVTjAR?GrsS4FLoqp6Y&DsikWsmyJ3FEq z$m$Crs>nIcPS;0o#wqB^5rvYM>2{q<<>IskE=pdb+3-$I>>^af1 z%AvIiSq}~GwV=OU3~O>cs2+vcaTn}A`h02VIYEav=iA&Xyu+wvVN+SUPz z2L+6k^o~vOINgO6Sky|1AnUH}>jJ!3l`lL>_lJzTE14@I83c!`tvZ$U#=99O7>}ed zRN!E$4$QjhdN_#9pIhQJ>k!A!M!`$1M|HJ*c4T0+eUmASHDuve&9W}LK_H7EH|BPI zWea<0CeL4!)*&U1k(7Lb#glIPKpXaLJ65gAnO4wL97WqF;kGb+rMzRR3Kh)U+z||9 z|6P4+x18DT?v(g}32WbgdM?F`gmFs7w-)26#8W&yhl+aS7U5=dRZRg;`=P?o8?E!n zc>!4+`T{6Eiqs5T#tL4}wnI8NNCP)Hu89e7FrbVPWUz(<1@Att2!bZ4th zyJ7l^vBjmk7igi(_A^i=Cygyii8kqm@=QlSj9;Gj1mBEV+Q%RhbP2za7${ykuS(#dFwRM^?-ovony)Y@#-OW*EKnwE(TWe|~~uSM0Qb z9xA6VQQ|0OmH7J%4j1+>&?(%=@pP5n30Ee{Lb$ zzJg~FM4Eu6n*5#%!inm#z0j?FQx!4WVy(&`6Q_yrIurTMDihP}AZ&13a;z9v`8BD# zHoGU2QQWPXv1wdc;wzEQ(5)D1_2*%1dv> z;T48qr+wRbTD91?$jKb0ur%+te%7Y1Ff<(c84F_+ED~Q1F-=uE z6+Cug);n>Me4OE;#{@ZDDK#V@vh0|YmmkOP7ARRg5Fb>`>kBBmaBvfOJZf}D zG$AFy4^d@aB+^YDSWaJyz7o*!2akjmZ_7ilM|PaS>`j6c z9+^{tft_gnO}i)V*GP{auv~UEWdmA2|8(C4bApth%MW108#y^BMtFGNS|~XI0Ycsn zSuG4q3L;tz{hC2ALpuCAC|Sr<&vX(Ap2Mi;(BBTc-9v=b4a10-0j?T?NqSH0e(6>L zEqi}@HvM~J2aJd<9uV;VHtU~sc2bGaYA*fRK#RM7SpO*bw*CSCx8@;ikAPxK^F#ys z__Lwi+~^c1BlE{8?UDno?a@M*lAmCnrbjHm(23li903^{i^s-YNWRtRN-ZNxqPa=S zmyfX7szGesasZbmL7a5yHUce|erh6u!4Q?dy=&MxctQdG{vyGY_z9_~8A5Q}55V(^R;|5bs2?B*5fw5$W%6Z0hiPQR3b=F|0M``1EJzAs16pgaas2pn$sP_}>wy;f zZgT0=1Q$B}qIA)Eb{8DY=0y8OjeOj8Le?1IrDW%WiovJzZufBVQIceI#oJBUB>8XX z4!BUd3UQY`4~JbQUM)OH8SWY_Kgy`RmtgpMe4}=v<0+q!;y`leMKotc@5FT*bzF?v zU8~y3T<%+Zx00+-@T(Ag8Z@MqO-k0wSr2nL&nfYlCOkoT7d zc|jn+b=eij%LW1gEyJ!rUcewPbhHoP_2#b*@_u{$z^=?!AnzX&X8o%pNJ>L0U3tjV4*I*DnV{l*g zSPN}m1N-?u?et5D(SHMG3jzbJk1ua={mA|PlNQ%ML0Cc1wPIj91N76}4!@H0w~5NB z)te+u?&zy7FXX<>i{o;wQ$^|#P=*-bF$Xs?8Ah88;y-#G>N_W#pXQd3p*GV2xrdMG z!7{-k;~t58qADp}+bl;USIUvUl(wYOh^KPo+H_3&{N`Oekw}B;0kWjRQZ1e-m=OW| z_hgQOGJ%83FK>xFUX{THJ$*UKa*#%XDX>)!|C(6=yx&}{Qw(;(tbb^2w(kJhU_Y*w z!7SSfLY3#MsoxlMfH&1$3JAH0CCM#)R4#va+gXEladgI3&Qac{=*2*9AGi7{T^*e! zqG}zJ?NI2VC!eO-m}UrC zuh(E{&5-J?i8CuiwCc>$g8S zhY11$F8r=|x8Ig`gANh`JHda-D*(c~fq~Xhzb5YnO-KT^>#lgW-;sBN0IgN8csGE& z8w6+t&IZkP`hH;8198i{1&!h=MQyj2EqStqp6G0! zeA%37YwREjnT(uaSQt4W?l;{zJb}X*=esPg0tB6`1$KI~LBXW&UpBj36FPKu?iB1V zfrJ2;xLQyUZhv7_y>?P;J^{z8r`Je)L#%jLsicFWKBN12K*b$VzRT#3VI0O8ulkh z;tEa#0w+Sb9DoM}*H4S=5#8v}MRu_r6*k&q%#kw1n3kLivo>fl4jCGmV1f}=p|&Z4 zDf3(V@_iNe@Y2>6>;#q=;&SZw^vHRPQMC;BWc}F4Tiq#W*4OK+wLd_rcOOL)D#R&D2Zz^$ zFH%H&l<1*uCV7dKtuQudVU;(WgdaO0;}KY{xp#mUL{~z3T*kh7ifHaCa#>^;2m-W{ z{fD77UT9<(bc`msS>mzJ-ue?Py?Z+iQTK6+$K^{8LJ83+aW~LH8av;;X2lD=`;;h3 zBsVt3S0SR(Shs?h&seXwBz-yp^5}mf)nfbHXqZ2D`lXLARM+SOUc1|HT9DSnF4E$R$Hch}! z;c`Qh3|KV%__t`fmxbLlT6G`R*=*7qvNaM8ixj`;fH)$k(~4GHFSbPw$u?1kfz>WJ zg3*dKE&=QEJ>6{_M4Ve0mb0H!c>-UH9dhwp&8MQ%UH(2^l}}2@r0@&WA)=m%ZzGpCh_peOetD=RQk_z@@hPKT`c{O$77aF8Q*zmR_?#bFaky5pjh%N zd>p;*&xmc8rNCYZ%WQh*O#f>YhmIXD(ZI|icrFkF6e>f6YN%rx9uoCpF?v+>K-}{V zGE&q0#qC%-Y@wbit?OV0{{ps8W;K1fBDT28(p|{beVHlMFJxXXz$fFZzCqa7gkjn< z4Z{DV&BXS?UqB`q`SlmuD7n4ZSb_KJL;EjIpmT zt!r5D-r6~Clf@`*q-J}477b@XHcW!y?-dt6Jea`fE}_t=t78Q*R?Y zHky$gt@(s|zq*ILHP^+eo4MGo)Tq>YSh`{O&PK^$*rXa~cl20V;RE~NI)#xe5v!(q zY`FvspJm!+&3E-`E&MHBj@Z1mM;|c~0BbJy)vr?Rw?BR2fIv6Q)2$6-U#T{uTf`fx z{Q`%NLLHAvD-hi{Kac>~fc$H$CLvXew$xa}i}iujp&QHw+MT;Qn7bZd`C#3=&iE(; zk86gF^ca(0E3q0$J3i;Np*ofj>8NFWT()&9py*CqLNj&;@sk)`VIkgQ?($Ade)gg; z1^o>3(1f7YCQyyyMD&a{#1J|fXtXmWC;OtKY^?bs{wP|-wWb^K zVKj!e+}j$lehGNk`1lO1k+!N5Bm$q$ec{gM=BlUE+|FShEURE&w(FpsU|?s#w*i&! zcimlu|7zQwLff`pUdMbz!|UI|zI^uIgoqJ&MBzr3(qejYF&a{m*FZ$(ESaaJG9K3eaUVaC_BS?!Z$7)OJSCkW<1bRBU-VnJ^uCS_9r9z z`KO=VaKdWC9DY?2NL|N62&uENL9IMvfh%c9x7yS7nv# zp{OHY+#OYq%YJc@^;$TaSEfuAkzC1&J~_!H0q4|WNc0FXE{53t(yatq`Tj$@|L>eF zb^sK#{d>&EKWXvC7 z3J#>l)T){&DcS$q$3 zW3p9le<=wWXpeEq#Eq(6E}?x$m=d3f(TKMpgT~kW$pROiQ>aUGqSpr#m&jFYhR8eCZkRb0E!uV;*kw`m_o6BkMwrJ%-`$EO%U#vZ+ zt(d4Q8HBU4>CK{L6j0M_IVVh2E7ZR)o`xrr*(i)@YnMLSrI+Vk=dRtLCh{h8Z-#!{ zDp2zi>(m6f%C*=?6!9LqxvVDwXJ_z6u)GkX3W-{cAm;r>K2cQ}vaU1=1<_4W`F)L4 z{qbf@w8{iTrWt!y865eCaHF-?{DiU8(=gRhII<%jHqi&&QrE1Pwq1Bnj>&7YbLlPz zT4=KURL(KW*V=pwj9|Ymj@6K{V#Bh!Xn4Uv@lo7M3pxG>FPPab_i>n>-JVhr2!A9}q$9DF zdbd)y!<*L^$BA|Iw%oOl98J@SmJXcP2l@MJ?$?=#-Sf~`5qK1&RZ^B*_D@SXjj`KO zKT$oW<7phzqvFWh?!>5%CN2|@KkOJLKeskQiYq|JsOB}R|I~f9pj?evKwpwpF>gfD z^$AC29UCVJCF5}E#F^pC#cv*U3Fo5 zq70eb#hlmv5+%+ea$^@CR@PR3(|X9ACx_=w&1`Q|?b4{Kt~)#}5DEWqkaw#*%nEK8 zwGI)lEjYhPuKBHQGVzv@vWdTD6Z^c)!t}yw>7t~4B7^8XoLkKfj0U*tJs{;eHm9kK z`5Z?BZe!dDHt--tLxe`~W22!3|ET5^o+M`;I?>rTi}Kj>ior~KGIS8zP&2bsDy^4tcc8dpGLtjP21C59WL8m=|-68+-lO-HLEAp$!l0WEV$qzoV|LNY! z_KQZugP_Paz_a-(BL4f?8Q|KnUn3$ODi;KJAY9GP{AXroIDxJf{Fx%+zYSpkyFUFx zBFXQT$PR*%Pr$bEAJ(@niLw7D>s$Y9oP`r`iF-B9@~@AxK#iTi-r2vxI14A>a`r0k z{7T3z+|^bFqmaZP(3jBEvo z#lW!V3hfN}d0U3P8%9fV3-h<5q8&%tkecgAUJh6i24k!hDtqL|Kd>nZ%L|HS-JIf) za?+a&xZh|zb58s|nvRgMK&Oi>XbM6mjp@4QT9;ZY$aB%JwHLLDXur>Da@leu6yO8+ z8h;9OhUzTPp~3oFiHmr}28}vyDPoGj@#j+{9+KWd)`HWdq`^i{n&b2}=JWTa+-XQ; z>-3$<@8$78K4Bfa(->&#dQ#K+`r54`X4sZPT`?!^G4Y-tRs(we?gGEDL>5K34ZS$T z$G(07fhG9_Xo|2erZy@yWtSfXIw;OtqjVvOj@#rtvl{&fG`@lf5G@Y^L3MNx9`d<=80H=Y-~ zMa(B(pBbWjCE=y5MtKAFO?18sUhk$9r%zQ~$z@I-JzrG&Y+9eBXL^@jGCA= z`d_hLt%l<5bY~!Zfs1Ff@imTqI$TCl$Hu_hvYmnAi)ZVTN{Q$q72wEfMSFAZSRAP+~!Ed-x|J=`XG)tTUM0)kS*|qqh=j{)r&O5P=!?Pnl@j?>xjY7 zgAaX@NmFVe@Oe4ff+#9cfj1s*i|iBe#JUycPB(=JG%BimT=Vb;103&fTuXmfA%XV+ z<4p?wvjC)-PfL02DvA$@UJlI;VS-#mlAk`i@LF}QvQklan0YFmN`{I+klIltCq}SC zyK^@sA7q!dVNBL{_!*OB@73O=hc(dp_}{Lo06`tcz|N?jAPU*AYAF7N25+ZULhJRh zC5PM($^ATc`dWf{)?go=+HHk^kMxZ6^$QOqM;cf1e4nH0eGzsNwm9^z;%CWk*Si^& z^HEI_yyt2*gNrm7%S!mTv`59b^RI|4XB9^*si-8EB+nFin)Q-!dGL(0LO}< zj6AU8{0}u>%MG+ZWrIEde?Qm4A0hng9W-)#$KyJA;8L5{0>66LA?xYICPdOmUJF!Z zVvU&?WxF=Lh+!l9rK!<%_A5fpTvg`i_z=%BiEM)OEOkLAZ5?~o#+V(93w7MgOfj>> z$Jebi!o?;r$`K<~9=Jh={5v`{*FSvXFY?-feT2D=GAI^Z(*F?hP)N-%A;kk@8RA!B zY!b9hrGCeq+x@-KsOy^<_&(-LiKiUTT}16oN81*uzC2vH&{#btEy7&P6m+`m>3|z( z8T_r!{=*))(w_j(pKt@Mt^fFlGZ?T;3-ogX;V)_I0J8$EUw`hIvC9jWic%s?xdoF5 z(@>=3dVaOn-DzN6+iVllheKAiKkm`5X`$Y|)yuxOU>5-@xkLOVUEw1YdeKXQg{^rh zf0XV<6$5X(rqhlXuUD|>IU+@8Gb118(?|?CgB+$i-K+*0BjFk_n`^FGWKgxXi z%8Ksq$XpQ^+3Ltt)bm^zp)JwKrQzA5FWo1@Rl74^8+OaN#4fpffLQ@ohCf|CWW(8@ zdLJY)H~FR;G1uRHK9iwq?LVKoeG+oBi`XXKus|w4;jNKckc>;%%Gj4(iw+hd(d``l zGezT>=UP(soF7g_QIn>9`|Nl4tgAH=f{JK`W2f@t^5kw@_=zgGqI=)styfCJ!o(fu zln7kW%Y{?(i$1hWU72oMG1#Nx9C#5dS%h1w&FeWh0gpq&I!xI@sc%U$Glum+Wo6!q zc)Gx7gsi_X#BNS7j!odQur)AX71tHV`^V>a!K^^*^xq+_9?S}~JiUU7e|e}F%nGz} zzJiKT*ucQGVOLNwP^cIT*e&)~hl;^~ZDdza@gEZ^1_QRX{i;wg7_eCFDvV54+ieq`zaXe?lWPGZ%vG& zWIYxgxV*I0FS5g}7e{k=ycb2K7`Ir>If{AgarHGi*Sc+y6Ou8w}VI_tP5orE-LS4c`8? zY7Fpt4_(CjzV*!>0aYxrA&I)B`cUs8rW%72Zzi2qMoKD;yXvzcMN_?t31V_r9?yAt z8^5*1O~q-5-=NMWVLD32bzfsrAWLzHG3&m;)HCWdd!+!=Fw^jBdiKm;L%PWe88}@S zX{oTBUWu*F7-kqbR|qRlWu*k6$QP5BS1;EnpE5 zQ#HcklEtRX&pSdVI4i5yrDe@`_7lc8qTolD$|z83c(DS@yKCV^d&gg|^Hb{eUV8NbS_oaSsedLm6%5#c_HSoX z!GLXLe|G*83|I>WzT!XsIQ%CVu-A<3r`q`!;XlEE#c5ak=Wok@f&m-j{&oKI8}NY*(ZqFwzhV*z5L3A`PJ-c>oXdE6z?#fP(>Lj3A3$M))-;0s%e%lLr{M zjO}MOO->rhA)!sWp*+(O5aXBUeZjJIUfEh-?;q(6L%6+WZ92j@lI*NOE&48Z*>;?M zYi01F!&DNdWs^AGtg?srMOLO3TWy_YjQ21uVe&j{NfUfE)>hiv^mbN0jjRgU(CZ*< zZH!a~+Lsp*(>%pBY9kdl&+&Q~UYJl3lUW<{j&|?AbVqStb5H zgTsaW3v>!May(t-cfys4GWh}$(ywR*d5|39&-gn;8MW?9^+u^wi?85W1d%47sV2YY zf^ed`Y%g?c-&94+wpgn&$i!(Pyv{_vv&zIYI|v)xmK-a_RenwCuFdYrWE6L+ChbC3 z_c#Nt{E59yPkd&dEXTgZx}=GL8@2WaupV&~D~qBMCJLc6obu9Jad?Gc*lFK(UirEJ zO8&toSWezcyXO=H7{PyTac)l-Kc*bUIzP zTv(8X3@$jb?l531bz(CVxam`&kt5cjIB*D91ipTJ_HgCJW6mmWV!Yd&DT=Yhg!J#9 zny(^lnji*s#H~!?rz|`X+k&6gHPglKwkYP(XfzRv*aY9BX?!2*P-{lFNmOKdPxj^O zx;wUu`-auk`fT@#?Mzcw3LP zbu5e0(59`ibt}TZD0^}ax5lY{nS~e(Sfs`VO_=*(?}bB~-IQpeZgT0=1Q$B}qIA)E zb{8DY=0y8OjeOj8Le?1IrDW%WiovJzZufBVQIceI#oJBUB>8XX4!BUd3UQY`4~JbQ zUM)OH8SWY_Kgy`RmtgpMe4}=v<0+q!;y`leMKotc@5FT*bzF?vU8~y3T<%+Zx00+- z@T(Ag8Z@MqO-k0wSr2nL&nfYlCONtI{PWn zRyKkc1KQsgbCYYjPW9<2e|RSSUE2$e;@McG2Yldb2)Rhu2n!z?k=pW_Z~4~9t)MQl zH!gPs6bj5A73_ExQwqD*bB)5O}ediMQ$ici|Fi#tTA zr(kp^Qfq$dvqf+<_BU%USA2drbtIBRb4JzY__(OWPn*9?Ch=Y2Q2W}eMf%da zo)^r%-gv?v$wcWZsdE)_^4o#w`91y3b=Fu+pKrM z^cET&qa`EEq$nBS`x#T z)lF*TlH7_)Msh|tT{OV=4G#6ue$f@SE8I7s;P2LKW#@9^?#_C#Kau&nSM-G5kEpz3 zn=x?LU)8>@HGbU6tZUn`K2W1fPFEnTv_!3});r7+^v;*%PB2rsV%;DKym->S#b9UK zDP_pXQ-9S$AmpGh1UwX~FG;KxVF`rA?ppAX6nQ|= zNs>@N@*0h$7*=OFplwCrE>aw=h3Rm!MjEI+p*9SbMpwrG4hgn}=F%1aDJ=!(x4-rdJX?XP?BvrHiV1 zBj4};BI?(7TKdehgJSX->cq5655uVFa*VTu$<_-USQ{c?b8@vob~>9E0@OBO6Z9b! z!X_)4qXYJJ1MUy`BugGN%EHov5Tjd>sLvKzWi>ubMfW^GsPh&o-lc01cjRwUB&G6e zE`GWm9=TOSe$l0$jr6*Z~)^e;N9V9cUT)`_o_0juEhX!_PhXt8mGG2$uu{cBfs@ zUwmP{z`WH-jKg!I175WRfCJ#Et@qJ&GU5VQg z9y*Y)ihCvmvq^HwPYj(%xTvILPYU)a^v!xW`XG>)9B~=-35UQ(r8Vv27ad&@tMfFa z9E`g)LzW4YlOr3aNZRp5gof&@&9>9MklNc1-KI6L>JyD3-Y87JXxb1e(wp5AuM8#^ z_i&FOJuq0j>54yuSyRrPG;^cliJeYHoXy5_W;u&zsZ7tixi~wr`-mn*_Fg-+U2`B# z?qO1TSn*Ezc3C0^L2`OL73L^Ml*|j|Vf`En1IVf}MeKFin^E0|L7=e_qv2<-_*=t- zr`k?F91AvWo}{oiec+}u)w=!Q?rqz7QZDt^2-S%z#lJEA1z2(Sf1tkr8`ao;N%{-0 z0_`gO@Apgp0|R%cL9epkUs{+Wr5$OZYvHSJQ0~A^fl?2$D z#Br@k0v0J27iZO{sR7D|E{wg(6%}7l?3_@^4-cvgDCLgI$GfR=4b2nAOOv@1Lg{+5$_gc{3 zE&^oo0K3-yP)r_Rzt|O%_kUsX0L#t(T#9EfU^&^(2-R<`cm@NujQzribihSje;UvU zu%+uNiS8Fjq66$R`#qBA01M8bG2Y)-O6*}J=#9{PT(_L#8)t1^{PG#}{$-xTB;2hb zILXdOLQTPLIZnwj_s13^H+r~f>sxM4$~&NqtVlk$G!)`)e5}A8@l3RwcA}KMt0HWg z{A*41RL-<*oeu}G%w-EnfK6dn1l+GrzyUU){pHij01L~m=#)PwodVdecExV|z-|DR zzx^ug24Df(RhHL(Cd&)BJ?GYrNRG*L=$J>wZQh>!Y()iV2|o0^F|un2ncn=LA(wKFhmspAb0|h&@{Nm|GXj(DhI`FzON6#f8&8! zD^yrPtO~59ko&A0A zKbUa^*n;*OP)`4Xq6!9X{rZzjRKb8vVpkZ|9|@xZtUvoz zL$m-3$gV=Pep`$R0$860#i75i!2Uk$0#O)XFNZ!ZmnpDvf>i(OQQ4O8Igxj(ZIw)Wgi`4I>}v?jO-BraOP<#uz@212ra=5{BfMH}ibB6P8P7GfOTRbJ@uFXn z<$?hBeL*A5zJCMRm831Nvms6OGgJv>!_cLEep|4yc<^;%UG0TBlT~z+ggT5Ny~&x+%Rfm zK|REKN>Zy>FHz@W580J=y<1Lo(B#&s-RK&Yj7x3M+DV&qXF!&W=_CeK zyuyx5F6+HRBn#J+z^7qONN>^ydkWlD5+*#h2dg~AL_}11B8YaA`@Dt7gr63FKb8vu z*xPlLuLg|ef&i9}u|YwX?~ilY2w45*5VyQr&?v4_)OLH> zk|$f}iO%-Pm(7{B#tx#8$;cUog^?5De$$=96F8i4zRQvgAiy{l%Kq&C<&UvR!1T1crG#ZLC!bcb(U~~QKZZP4xlUdBj8{+Yt)F?Dhx#Hq|AK|-$M@-%cNjwetC+6H(LX*p3IXh~`W?to2ypw+ z2L(3Be?^Y|^5iH4u<;9Ym7NG6I}rj{8g`|wMqz^h_j_GM6#|Vaga9^#{ncS`2w-8^ zuPQ!Fo@^fm;zJPJAj{Cx+JUrE}wl@$q^^-j{{j=t*h zLJoiZyb_mdohnj~fHK4Yk2$!J$uQb%5dYEhP~SP>{4}?O47HgK$US^a50(iY8TUx! z6IDs++GaT-xl)e&rL-lLMm&`x*QR6I=Qr=-i9{My50E7lmTK`#!HfvtzbA7PlnESM zetApe@v00q=;_N*mV-1BOo6R>_}9z|;Qi)mono*PX8l8Jvwa812K#Zf3})F@5UM<1 zP5s881H7s3Qb5Q}EJ<$ZqjLGX+s+!ai=#8Pa*pyoMK1<=`?%Ft>FVe-5moD$Y==S@ zJ^3`v#xz5?G*LY&S6t*sxGOa|w@?@|e8e2U$zyB0`evhMk1J9<(?6}Ae7y!sYmRhT zs}s0L>d)_gh5+|d{Z-g_2wha|to2X>X1?m__wwBgEys{D?CE!%?ZQ)ry&DfFIop=`?lK7#T@>U}trzn- z$An1s+GYg5<*xIcxqF+=v6YvSPIdKuBHF97(08d&@%A~Ybc_1*W%oefTBtuczzqRx zi@E~1|3CmY1Oga3{i#0xMRG79fPGI_0QYYUa6^E$7k^8D8?b=sUk~GiK!6TY|6C-f zhAX19u%d}xJVDV)!|IfLo}8<{m05eSlbl7&L~s@nW?0ROO>d6Ql7Y7%tjs9tLc9s@ z8qhed+@{M7&`Z=7?KmYCjL%)U-`UT%U#=-gDf93>3{|x>0?R#)QKZnEc?_Uf3 zoWK43+t))s{_SUReE&25{<+`2*SC+cfBXDzKaczSjRvSDVxwSV=YVFF-;pu2G`4?C z!N$f0)f#;Nh#BjKs-xTmm z_0F|TsNGnlvrjaOipYV%bi$I}*afgRlsSleP zlOrX)AQc*76&kF3|7v~x{5YxI7N(dOhH>nb=5X|Q;U-+;^NWnsvhA@2-V~4Ck?TTJ zu+};saz^jpZ7rWx`oJrjxI0X zC=AnM@bOyGdo=Tlg)VuReMFd!epoLbGNcDExw+nOny1WFH#HVK=DocL&jyf%8rtMx zuEX6dH$G7n|7^vFL@^@A9Kdt_PUUE%{X`&aacdP1-6ziTbUV7ctyYHJ6CqdIj4Mfi zzj@Y*|LvNZ@=Wxj!6Rf{7z@Xer?$q{B+f8jZ3JMv2unE62@7c@T~i1gNOUoCUzv`9 zjhjk&6odj+zc}S+uu_aHc3OJ1;D06bdbuJ&Xk4y%?`+3gNjV3!(^Z_8G z7OFrYrhOqnb3bNlL=x}S++1Vb+g?V2ji5E}(PCIj-zj~9;f62G{Tu6}o~TJm-MDIE`?_hh}3cFV%Sw!0>c~)amzn(E+>@laLugGk=H&sCo^950K{ z(gMu;1A+d=;5*G90va#oGLI)(?jSyUgdzXgs*&d_y7Tkc>bb_@8pbvK19fM4JJH?p z?L=bTvM$AK&iT{SXZ`3+y_3ZAMwUo%Wjy(C5^v8a>S5hJeAGWi2yH#x3aZ@Ev)V)3 zT7W6Ihs#w>S*4cLPdpq=B7_2iG|)+tsDEe-M-~=;qg2Q3tqjEtj<)WbU1=cg6{CgS z2;{_1r5FjM)yOBhxP{)m1J_yxFq7svWXG(0DIdIUdb+4UxZ;H=rNeRzc}3teT8v=n ztBo!lpcipL$Nq(`q&cXq7Rp|G9F%2HT5bIKRl+mM^ykTd}3ae4w+#EKVfk#b;jcbN`>`(Uq zE<`xM1HA#!fM9H;W6b3x$7ghk7P<$gPVI*7Qt$7KziuHNCVZc{V*zj#!Td(?0{li+TayZ<>8;-n&8c(9oGqn+GPm>B*zO zO0sQl#CiJU=hrs_4vNvq{8#7E+;L#a*kCT;i5Fk(w1j`8)byvBsSK8>GHA&e#<#9mW(M z_9ucuHHWfQh=cOYWEX3H z-VJ66wgzUy1(oa?7bWA|U0AH-$s^sR7(^RA>~MK1nGD@d)tI8iRO;(}g8nGU7i5gK z+Q=@+TJcWWbUEG9EKXXpo?ng)XOE{aOOqSEICPkturoGbOuT5QA+W7?_|)9mjh}DJ zrf{Q%K+56k<{7HRg`0pytylTSc>76{lj_f_Vy8+T$o@d^Fm_9{{NB`#|D`5a&8Lkco9?&J$=iq`7SjbE<~BR=jF?GiOuq9hp?EKLpF zX%9MIJ&LA1a++ya_o(58eB^V#vt48S%+8s9jU;%9n{(#=hqufs+cNDV73D&vNmywX zJUkz_o-J|LQn~bLir6JH@wD{y4UibynkW&6mxnQBTO`pxe;g6j%{Lc=4T7(3(RDP zI)yMTWc>qJOq4HO^ei-x`9;ZejzwEU$@bB#yeaz!STWGiTLyM_2X?VulMoS+5$VlS z?t3Bfvn~ZiN`xDae^BRMg6;20Um#ddPmdK973G3JVrupplB0$O24}}=%uS>XIv4M9 zvU}z;OcR*q6=_xpA&>PHC8}wua#v=!%LUD;Z83NBclX*D*CyxR51ELeCq&QC9{5B; zi!_Vil&5+BbB!uXKPd)&Sxv3P{m-sOur;dH68A+p7xQc@1L)=g^Y=S8Ci`_6*<_Y^ z=$eC-KBm0*0X&K)>^wF3n9YoNEyJ z%c1Fj6m9mQp;?901zU%kZb>6zPIY7O4n&fwYPl#xlGUlQq=%ii-s0#VrLwgYSjvkO z)mE;?ImRB^-*jt#+%i7?q@XnuuV!2E7f`=k(kmi5d{L89{YDXh*Rn2;0-QHa%Cfb^EKU#i>ezo6wsYeC;`Tj#mL+4L@qcJ9Ca|+(*4*R5d=VxTPwoW#;x7}KdrRe~V?tlQ$4 zgwV-NJT$axwczAo-LxF&@zbv5=c)o~82JxhyXl5?EfdpSmIT9%-?rP?$) z?ONBIkRQNDSeE#b!}MTtba(Z6lF{>Jwz^uE2?FZvXE^Y&OC0%)Vh{4m4IWSS?@y3! z*mN6l-yB{o!3l1*cOg}tVsld7NpMql3k#TeTYmmJ^rLwp&EvL}G5^+C9C*AG>klb$ zI4IldV-sIX358~dCq1Gl@;9}|ZKx2B%F7#WU!y0AVWq!T8r;JeIy7_`EmpK({I(D# z^zo}?gn^Bh{S=Q`&(Y9y4+yb+6ODD?@24*4vEKD9$sM1HsOg*O*>duo?mk@Yd*+}& zLn_nO^g8Nc>{3SXwuVAb?IAq2mXE1*k$N7{3@RaC$9xmP!}fU}xAb7g+lQ9qV9u(G zWDT^*PEmfC_2nrqADDU^q_5AT;c^>PTt(V1N>^xTPo%T0eL$@Jjkm79aDKHy19EO$ zEM2O({o0jEetAAqdGokJKK0e=EId3uwmI`tlZV>v230$$478M_=3FH4p424X_*~dC%8ig*0rxM5zE3SZ9X2j*X0qvG zzMrUkxjxJqWrB2JJiLwm011g{mrq>;!OKUw1WvI7CLMl0w&fTPTad~}kjEMM{OekB z)RJwy#dVL+)o>En=2A~f7C{t*d)~}aOEx$HIW7D=Z{MF`8kLTQ<~vTC&RGaP&UMt5 zEcX9`8-N}_ieNhMA?N0c=y{mxdq`Md2Fe?@k-M&N(-E_l5}9&O^Rdtnq6*HurL)%_ z5t2@mAU$Mc4m;uZUehse#kw0ZwaD1+MXS>y##rph!gJiS#ACbjvPeD*r))GQ)zweF zLc`(s7Mk1>Oy8s7_}4T06m>70mX_#bJhJ5b@8NqD`=vhW3ipAzFQep96K76ai>#IO zmfgh(^s(Kd3g4u`A<3aA7Gr&TFp{0;ZFEEGtxcC{4E-(h>}x09mgz@IO2YBOZ(jpwpR(pGxj2-LRNx)#)$p{T zxJ*bGPYRw9l^)5FxHo$_W`o}2o?yBk*=nDyU&y#BFGGpbRQpGqLng;rD+wB{v8AO~ z*iwAU6v#f=O+t{iuE+KR4RFZjr6CwRs*-b2Yny7V>l;6tHleb9it@H z$|Rk+kU42S-kXdUiw0%)-Mm}sr>umMu54p;TGsi=yoZIb^Y#|}3-#Q!yfkUXwaLVw z#6*8WEjZ?_AzI;GrClYZSkGN0T%26#?DdsL;)uh99d?~cxMAWG1(X#J%jvKyyB8UI zsW{B2Zj{o;Et`00L_J)g{zP^3bPTbf%yN$%PO~p`qCU}3O-T%&JA^K5;>D2@XUS!vvyB?i?dhgYWab9ycyAqj$P0}`O-j8nEptKmK#s9x@ChFp;nl8}HX(Ro*gbS=skO}$!@1ir@v2`= zR`-PSJ6t=t&%Z1!uB&sd0M-}ua) z%_MSHC_(0v*|Fk1FUXpzNV}4a1fxK_4^qKd3Bs_9UboZh&2EMB-LciW*UFMWJ47%l_WYx) zl4|UnU@nc0&BVh`p7M6B@5yqq-+M@rFGyg@j`9f@@Bc61pl>OMi_Qx@+%Nvsj0Wp%>y z>33%-+_ZVUofb9|#o=^XrytZ$+_o#D$vrLh5YN^qh1Qb31`0Nl`{_BWSG!drzIU_M z-5z?_oICmdHTLH5RDV(1aPy>uGE|CW2#Ir!GpRI^xl$QYi4Y|+WQsIsFc+B$l_3?W zsF0~bWU#befBVMGh`nGUyY=CEo z!W{))yKN%c;!9`$nQgJIXlU9s$2E?cSyNp##w6oJUL7dOD!P-o_d#7&vu5--|GVa1 zvB8}m{92C*U7Txe-t_dV7R&0;_pW{CN;_8P72Xyp%^ZxoEE2eO=0@pX9oo;e!{xkx z2z<}Ix;@%|&)wf$9omT%^7Q!A=>e9GI*sYGLS08}&#iO~%U

_~k3-^wZA%lSgyj z{?YfSdz#z3;(im?`}4=<4_Q6q&)b9U2qskKe?QXv?wDhD=z^tf^K^T>%3W98U3nzg zp=ej@!zUFjSy4N;-m2BM+aFo{XhmLfPKxHP>n%zya=-o@6A3Dq{%OPF9g@2eW_RT{ z?`VM?teEMCrDYjeJ)oirdpdDqG zxyp8{@rH#Dj;#xQntrG>=}_s^(9OTUv^gaV)x``NPk(H@F7?Mc|DU@HX@6Qh=;wo5 z85(Z|a<=~T+u$+%x7^zK7E^mpd2LuTDPqaM?5Q0heDl5zUGMg7Qd+%?-*@Ginz`!A zi=|FiT+KhQ;e~@o?39RU&o?Mtx+0J21yHM<=gWG1kso@BZbzz15@Q>fkB!!?aFE$$y@8&8I?k zFPr^h>F3mA9ziE>b9Qh3NTWSudXEcZ@yT8m#nas;a=C zS?+voBd4HX&z=JRx$XPJ7L9yfEynwxX((^MSYY77CDN>CM^savgyPj)wMw&g+M}jH`paoW%+bmcjpJctXFE;9QRWqZ?f1tx^#(ZNSFhUgHn7+1@O^!678Olh;ow5^3Y;XUtGws#OwngdIS_(1#lX>!^`8F-k4XIN; zeqiMF$ruazY$1lgrr0dM44!()1&LPIb_Y-OzEwTF{=`(N)B4-CjWf^bMb7d2Bip_^ z%;}HU=-WM`HAhY!Ire#NdX|;hy|D3~x){FiFZcLF+_s!n9>o7!YmTbf$$Fi+$}vJa z%48Y@92cx~mb>+cUus^M@}oI1H`reV*S-1rG}<8lxbNfgr*&o5QtXYB?(KRY7bj)= zcuwg)2|+vO)hAybu(}W_c;?;ZYbO35ecWCdJpW@h{v_gw?mZRlJ$=81V;(Qq8m#xx ze`M|GG_U^f0+Vi;=FY91_P-8S_j!f(tInNO(-oZgw{89=zLWPFT^(}w6rqBAb z3VWP?CF=j38*G(*ms8Ld{X5I}O4jXfbg|Nr!FJ2XZQqxzlJf5yPdEGh=j=}Js9zCh zA{HbYuFrYW`~30l=ar^kV_nDYeY|-leDE!+`FrnWHQ8p(Ri=3d-`zc7()c`ZzPeC= zU2k^j*%24foa5tB!j7FTt4kbSJ1f{4?TbolB z{UA|7rnK$Ht_-^w78%Zsvf(kGv&$Z>YzZzLDa^n4Ly-SkA+N)=OvAPg-<5tg5~EuR ziwnhkElg9IB!~SQm-PGQ3zu%&;;C7<>EY%c;Ss-JU(fRwRru=)HwC^t*g_jzx8rB- zPvOfsiY+w$`F=)H!~O|ITa^8NmdtRtkh#s%z*Frvd+_ecM}F1|*R`x=1t&{zQ?7Ab(7c;z}3aYlQ_Z(+AU5p1SCy=+BAUQsgk0qrvtS;jdAYxahXOuvH?+KU1?& zRv>+)zn=(ymJPq6pIbP8k5JTywlflC@0F*OIr%ndE|ieuowapZSbOA(Dg2Sfj6{p` z)1$;Er}`(|mfa`&E537W&Bpn+lCBEtp4|Ta_}SUnb~&jg>0z`wpg#nQV=X((Z?9Tu zVUts?Ughm77y6;{&+y?HBOf+jh*ZmR|Fy#IqRHIE2b1zruh?D7cU65C{(yb@(v%%@ zmi%;%b<& zOia{C@3^vM-H!QN;yqoK)kTZbndTlV+TPY~xbxBK;rpqHf=)+$zf=uh_H>n~RJAKL>$maMT``&F}a47QI*C|4l1?JWAGPgW03kC6h8Cv9I zB3HJjvvg#-;k*dfW7e|0wuk$z99~w|iPeZX`7alHvQptgMW(`Z!_Jn&evTzwuP+2C z^~(CFAEHZVujtZqPs%aP6L(6BwhJGc`iuXMJ0v?+&vnf(heicUFvj=Sk1NN>`kp_ z^cT-fn<^x|oKZ__jf~5(?e{I}k$ap3XOWSC-E5;s3w84b}XR7(NO55@+>Rs5%P4BNnj3myxdG4d%_jAW%tc!N(*JPD_8Wi^Z8JJu; z`gw7%b%}!Pw&K&LS9nR~Loh*FJItjcDCiK^L^LQs+1w)guv9B zTbd>P76tm%oo?lIZVPVP6km5HJmk0U@9(3^*YD^$hObF^`<<_}vc3J{#|jrO=|z^l z0v;kWt@MnPtK(yME1I0leX~Vob`%fq-H@DZ<781j)y6cv!gQ?RrM#j2`^4csZf@z{ zqjf*Mijqx|xxe#|j*ZVK7J9Mnv+F7cXReFN@EYDV(?VB# zY3SngalU4L%Ix9mIL@t7(MSnYxze6%b|IvHUU`tf){L{u-d?)hvLUpfU%d44v`LFK z>!M!0)KiSoP_ke1WpJ+D{zXfF+z&`&=nP-oIVCpi{b*wLTE6d3h8LOy@bk`Xc<}di zeDM3udq*!XsZ}V_oId%&;8KrYdlszosvdSfa&?``$Ngm=c#eEK(Pu_iJ1m4 zpBlL`Fw8yZG%NIA+AX0E0`-!Ys=9~XNCq2h4~g(Ow|e2uTKy-9J-VNrd9x2DAIvzr z;Lq|zf}p7K=>Kb4YVRD5R!-f~t>C=b+H;hxNYbBCR%{jMchCFj(m ztqY|BIkj#Hmm?Sd32gIh6*So7byN0)U7-z2;bKFNdBWZc?kcZ0&X}fcoGDNz{Ccl? zjx9+=<5?(P+QvX`fUG1PFH9AeYj&ZF=!U}~$ zU4B&c&ui!Zn)BrJv1Y@#W!tJ3Tst2$IxA7sJ-n-7dVjQrsaiqzbiJgsyLoZ;HeIhP z-2;;k*IqM8yVh?szRAq#akg1cX6v98{Tczr+MtW7X7GOpcMoT+GZ`u$@67&WG+n`nUyKi*!fZT zW24U!T{}^Sc$4qCrLvzZUccJ1c+*wxJ;`44_hHKWuZQ#6s4tOJb4#~!oU%uSdt0jQ z?6ua#?4-i875x_cLVv53HftY!XCqxzJ?HUoeQLJrNuCHCJW&JF0cP-aURmzaDd?fBLi0-`w6R3ysUS`KE;b3_rH3 z+D69z$!n$Jt|FrWhmBiyNAs1M@y{&ezsz~;*6vp?PIuOJv2*hk>K~AN z`PFx0eGA`n6hGC&D{%mPcFX=dJCoc-vpQ9COLm)j#SD7PJr)0(tMp=b;*j2%4?~%| zpS8E_amaAn-u5UUBBs`Us)}ucCFj76p&p~>2UgXtc;nw;*74(+wH=dp$$SynF%?VC zw?9u09$c~Jw}d*Y`Q&px)*R)9Cf@~v>YVgsSMm?PkPaef!NAD>Rlrb zZr`Qfh_P}$`efI`^njlmSE+wKzj^K+`p`h?&ZL1quD7lZB>gNlNpkttnL5O0Q?`Co zBjBJ>a&NnIV!(OnN!97i1y^rQOP!Q-!T+~l?A}j1+GX4vBa0`W^pTNnDh+MOy<)YT z-oX2fv-C*pr|R#*?oFjD&OA9EIXU{BcCU1&e$SVKJqx}2>gZ3LPDF+u^IsleqdELT zO;_7-v(~f$C7r3KfAhckU0vj9Cn2Ug$_WV`W$XzF=ev3!!1A|+XWNke)zvM^GS?(o zXKRgq{K|M#$TZ;^bu1`}d-5!x`_i%qJuiM}I>OH@`2pQs` z`+ev;9OZdxb-7)tJinX$o3+V*Z#iBV3BTYW^JnnOStsZWf0~(5REBPoeCX$jeNjFi z11=xcJP}&*+$Tw3R>T2aadp~eNp;n;GJB+DH^kHJHtnp^(B$o~K5ZMm#&h}1y2UXb zuT56Wa57l+W7c7s!E1&{`B29dPb+4Pjxxh0NUD&%SiNenOKj%0KBH6lf4+4ogx&98 z+o#w!Jj#5PeLMe&*#QUs(yPA?bk83e$xpw2#i!4EY+O-nUzlm;uP;H0>$C2Pd%xLf z>y>@Wu0ym@qU7vh&W*@1`HbBm<8epTEL^Lei-r_L+oc_t#pbSwPSE_qS7PUJNucc0 z#^-$J)!Q{xJKhFVlq#QlX3L6*ydTNUHNN-!z&zg90dM&Z&C`4PX-j3%;*hRDi)+2o zJJ07yyeQtZ`l$WE)sik{?1IVLc4?nIIC|7&f8HAZ@j%`2Bj*=x5nCd1@_F%!mj$W9 zm-NdLWwqZGVWi# z+c8u}uJDq^7M;4jrQD_Vof+S<-Sr+_SeW5(sO;Rr5PhzGqjjA14aY=%&8E9sZeNiv zmVcwQ+VQ4x37K>891Fk=G_jDb-89czIJ{bNEu+R z)?UlAKlG9Ky1pZi>!f-cOJg(!ouUippHvwH)t=QJd$qSA z$IPtY?r2cJ=(^=Eb>-jJ1a(D5bo9EFS^l#xS2SufuGMpY zc>j44xV3(|tN~VL_93;ZL1(|H7j)7AH6g%S%^9 zn!7LuZpqm3H{4NTY-P`SIkdgP%3bN!uPYJ}J$Hh3z2gbclrnzIuGUR@{4S6ud8dBI z8w2Abh80e;erU$t-5t&wzJon{}_UMG#ru9DR&8V5@QgQED+}uZD zH(fLjRg{%TaCPn2K7!mV$*2GsF-?O|8^<>nAC}%ouskxrUA!dO@~&0RtajPF}Idthi0bw8eGNLqq-Df^@+} z7H`Mi#-~Od+vg&CZDY zE_|h8wh%W|L2cd<;iYaZCuIBC@WAx0|a^?l$YN z-2a(-T1-KPD^X^0w7CPG<4a0N%r+d$~N#oil7F zUt1H~5*qj7N!j1yZ&SamIr+0-gOXR!X8SASI!$?<+q|D1N;~SX{r*AuOxIs4e%bG3 zw!L5_eikosHoTntUad}GvtM&&vqsfohy1|@4(~MrM^}hEz2wBbupx+>B-yFOn^e8y zU{>T+Z>j4#ovj-hg|#Y#yCw8aq$y0+VFU$N8aQP?oYxe6QugGCSyUdl> z|Hk&W-j#`{a$8S#?j5Z@kw3_6J2Z*2uJq)->fM|9*WGOU&6=LflfO&jiExmh>VA## z%6V34MHMGkZITyVKJ(788)sv@-%Dv{%B9PO?y@wO&ThGVijVJrZd{E(>ZMHrHK`k? zsB0T(x71W<`Re-mNMBXc=IlsYL!Um)_4JYtAFYKO?VjuDo>S|Pc$p;aZIgLtR=JQgQW2>QLWst(>BV zk<4rN!KsSl=I6WDdkmgZcS{#Ov4|D!{xw(f#Eg-L+j4$-^?%s!SmwXY-629aBW=;J zN9m^XelGU05y$rbemt^m|F=|O(~*`PimEAQsY7-{AyIGIl=ikYT)lF=*zmaiW~=XO zz9bGc87SU~T$J&5f%<~x7-P|;!+qwJ%lr2zR+_DJQ5=m__`F$>`)OUbRCaE|eT5sb zj}6>Phb&dwy-Kx*4{I0Nu`Y<(Cr2188_MhOdOP+oVo6QI`<|i@j>-MQAA_t@-to6; z+>fww8}xgbb6v@x_uI08*l&&-6$J*a=5A4ZFe_2KMqvNS15u~mO_>vQ?zM%Zu%yw8 zBD+UN?AOG7)Ln4&{o`PnZZk!fAF598-3vZem3nz^@=*4lk%V;f}3#v2a3YQLpgQC%|HQOy784c*yGGk>(S-m3TaI z_bW#Fq|1%Es7qFVRUGb6IIefnH#ych@pWm|s5@7$Nc|)`Dm?Gi<=(1thx)`&o(h9o zlU}Vq_U3?O{`MC$XZ1<{JT;!|k)|8$`gK|T<$Efglk(Y>#O=HKo`$F6xlqnqXn~%V%o#TBpzjs;rqIrXuT=Pnsv(Uo?1Q zP_|WD{b~6{m4mS~i@ir0k6Ju<8z{C|ZC&{^Q=Q}I16PIQ#)+u;N!*Jo+yM0Min4)o z%@;DX8stAa&i*Qua!G0Hrfi>RA2F@#8f`~s%Sf)3{k*#Cl=H4QQ=i{2Wdu%&e^!+A z)O`^lxmNC`-W<*H#E-QbyJWkLcL~>C2yxr)u`m7Nf-?CenHSk2Hy_s;_||Jj%Iwzg z2_4`!IPfefAtK?e6l?b2X4YCe%d=0kas^`64VVUJ?_Lr*oVKVTaqUqd&*nq0o7DbB z{)pW&-m&N8%|I>FgVP#pHwmy^@3F*YbUq$6tS}Wx4N)@Lq!6=0jG_8aO#anQfgp|6 zT7F+%?FI!i`LX3w!!=iG-#%Wcr!`OP@}&nS`E53sH>f;+<}gJ+?rqem2CI|v4?Lf= zb9?nF-LLO+-le;@+rHU5YN#bR)7>uf#ZUQF!QPh6PL><8Z@zX;rQi4?nUqGCO}wkz z=aFqS;_UlH$oNT>fXmz0`Fgp3y+mU?zRc4p^bylt=aW6SX6I0$wm_1>r_fNbZF%!1 z{hhBdt7PlHE{MRh8~?vD$Z6tCCM5a(zbU^AHryco-x9P9BYeDvmeAh>waM8Jq ztp_KmP8Xk7>@trd#EhQlHv2~N%dG0LvsaA#I3#n@ZnNKS?>5Ht{UggpM5k@|8MJ-+rK5Ij zNe^SnX>+1WZ?x{JsT^u^y3@dCdnUnPW%Tvl8*-c1>8hU&wa=UJrGECzH~Xd(drfMN z7#{g6`*a#_w?M;f?Pp`11GD}Hte!N(IeE>Eoz+9!-)rL+_&uYm%v`ao{L9*Iha>8S zBgwRQ9wXjE>Te5XA6Rhy!Mv#rfgft>+yMXYWYp$sT$VleQz8MZNf5c@Sk)x zpZfIG8J6vNpXCawGiJx{3GCkEy8Qgd%y|Op5B|)rRLohKbJ2d*%*z-4y#M}w!DHU6 zpDLFOnvd|rrnCPV&m0=D==-=~dfPM0ocw2l3!;mIpAUR{VJ&x)UOV;5S=WK(<6X%{ zcTR12pK!S`-`?NE=R|fJV6i>}OHM8Ub&!#hqlLoZ=4!QZPFUj@setw~$A~5-0;SsmXWu?K> zVqeZ@o|*PiDeuzdnI}&z_*q`P<8$e|PM@=lj1oO>pF;!D^{3Zd-tyyct<*0?l^MsT zTO^FG^E%mNW8Goj9{V{{eq?Iq{58kK%zyMxVprM!eBl@yxio62)n$6|)oc{Xpjn7=~v#sP-@nd$z{7=8m@wtgSIDF>Qy%^;$euv~_e!RGUM8-YU z?UulToH;ylYi(YYU$55-@iVmikiK--Haj@f%COe=-NxlV^4EXs+CSy{Y}@B%x;xki`lukDK<*3KVYac ztSQRx+ud%Dn;wsMuC8AqI^?3Y@LI%eN|&$c(HDCK!;!Mryg`kUX6e7TuB_;E?o)aTd6V^Ve8 z_`W{eYIB@7B1Pefx}4S=dAT{-KT{G{+-3w?$ads(q%f!ZzSxwUUFlw1%8hY+arA-o zw99-z;jZhiaPaHHKDp}NqRovkrpgm)Y6u6N4M*i&`Mu<&cy;_;SuYeN1$ z$hYGc?CNDZr!2i_=xUpL@`J+LX3=BC9(H+^oUx{sSI>)|G|{q7RF;K*yk@`b>#D@f zHTg}(3p7OS6$T!q8rS5RzjpB4$Wua1aMi#3st zbkA~23{Bo2lWvvxwEL^i?wq~09p4=NIZZ;1V#fJbGNb})Dy zlfHJSa><)!eQTMs9wc&&74ul9rWX1*&h@OX z--ZYJ{E~8KHmQB8>%6TYt6;S3C-b_{p!~1-`|kbhRt)#=^!~gszOMLIweI(BNQBol zjTC~dvrShxWmZCMXKZZM9)Sl5)=}m+g=1a)%F->DOesq*k~+Qco#1Gc3cswDuBC6J zgpTIUvR%${s#3QL9^Ns$wOV(!Y{(U<%HtAMTdI!*$GW!%xV)=8q|BSH&NW^pQ-1J==)SGmJ9sXCyRy0Wam&cE)%^Km$9#X56iBr+ z6gW$4my7lM@?&%3+0%z>zMc9x$@MQS$?dmllG~4qnmHpEcRmX6Z{6Gfv*c$?YnMX| zJ@3cUQ|+^4x9WtSYBy=g$jpEGL1MT`_ekHI#-iS%u6`%p$Fx{y@6>s;rYMc`uBEWJ ztAO*zaBCyYPZ@n5VUm3{Uejr`(PDXA=8P39ORiUHLq5 z^mJ8_{=@jASq)83_O7y)Khi33bo0@@evVhtPHncGz3^h?CoHx^9ZV-O`= zCaqnv$7Z&J58rZyTPsi0KGD#*y!%eNg1|ji(M#i_A@>@EGP<^CpG|!0n51^-p_{_d zaQ^Zg<=4U=6)9cV|LBYQhU>W&#o?DX$f~ev1f8??>^$#$NS^V?-po0Yd!~D5*B7f_ z1#@rqDSVzCcVdNWaB#eXntbxy^Ty6<31LGs3v_mbJv?JGnVBuYKAZPFuL>G|QprAgIQL{A9vf@lu*zj#}dbnFAxpTer>a;f1vo6@rkVs{ps&| z`Gd31MOj|&*c7T>`a`$k#tyB`hr2SjUOjeQdzq&3Lxt`=jfx}1cC#6(LW7y1IqRd= z7N+e>(HVZ2HnqR)MC176j&QG&9s0&5YwlOo{TjVeIPJpj@Q{e-E9I&^k0}V0i+fxc zqVHmg&x|Xz*bqLY)S4cy6?`;QDmgmZD2KCXWUJKqVWoH@xj(It?S4%1MM}-aJrTyX zH`2otdK(0+qF0E|dnwX?ASZs~ljZ~IhqiVce4k%7&&t}|<8hKp_S1nA`g5MRH@dLu z`D4?iY}0i&4L+3!@V^`D^gTA`Qum^3BYSm&%Upu9nId}hnD|7~;rX=oSdk?kUM4i$ z>ozDcuo^G1iU{St(+_0F6=?;yKKe0VQ|A~x_v&M%(_PVrwQBD!>Im*QzsMw{EBYN* zKXBC>x82L~dxEXuN?K-BFSkZknOUCMJVz;Jvt5d% z%1j0)pyJa5`XAA|%ig@)Bdf!z*qWZklcMu&{!hBYU9Wj}ZY)c>)#1*{JxupwF+cS1 zk43vRiq>%xDhhRuiM+YRs?K)1Q0UCNqE%G4a&f{i-E`e|eWPWUR(!ircxCv(Q`X~k zdmuRLPkC8r)qnO^xVb5YN|eTMTMN3HslreM4J!|iv5l^u$iCLJI3YgcB>2$jo< zUOuf=RAFv6zkyxJ6Ppav+-gU0X>9{pCD-er3;g1m+BXipmu?RqPY|8^_HfO;n9ueN zytkd&Z-nj1Uc9bvHgDhp`qIA0?O%>PSmRkgN8>{Fi%?Mq-I|vrg<2|&{HpSk{mwhe z-92-6`JMA+uNTF(1?s4vp;5rwTr3W4YQZ-0k0xjI+VVyeR)$c@mj&cc{__* zhdbw&-RmCvpg5E~=Csv9X;I+qL!#v(od=@Nij)o=h~B$?WAsT{dC1JgQJQwU24@J` zM-~jp*$MIfXSdfrbU$d?{QHAW8yhrvY5Co2alJfM144P;_tEOOw>OX zgL+wPVmU68`nnH^QJ=3SFb?${D-uKN0XMTbWI37=^%U{Mx*&c?hGXpX{Z2*R3 zlBT50BF9TpVpAU%oLCn?ay@7?5OP$GO?^m*sEg;BMpvRf?>w;_T^au#oke|?k0^)b ziUz|)<6tn+UWMI(YES~>(W?AztCx1Bt!5(KU!ZporU%Xoq^|t&QPXavYXg0gH64iH4%fa zxRNqXhpM<78_6l1Nke;!&QylSKZ!P&9O|LxiS4pzNdDkG8>Fx3pq|n1u{hL+eJ1`M z^<)LYXlTvpaD<|HVKdPgMQ5|PXb#vM78(af3BFB1^o2u5`-skAQg5(N>@$ar*B4X{ zS`Q8fzE4EdvGnB#n$NDm{JTq8INbW%!;NN2@qqC6#=O9`) z78@R&Bl-th8ugh?!|ThYbI7{zg;6A@u(PN)fF{fVeB%)55-`1yE`bltAzcSw2t&3! ze4z>1T=114HmMKbvo9QsDI*&M#0BY7IHl3t!%OtYCWP15ksQD-Lb?eaV@5gx-rJ=< zEH-hj7+iQmmB3&UNI8WUACX>DhDVgpctM;|UC_I@92^#8yBrQ~7p5N7k#M~;`N#Nv?cGL<;UPoYdq)Go*_sl%9{ z+sQq{gzl0115^Z(T_zY#s4p;ms14A?s4t+4$u)wNMEjfx2OO%)pds6l39?Jd8k~6l zsLMk39}|`h^_k7ZeF3usjTdAR^$%ncts9u4XionjlT46Fay?jdI9Lf;11lTJ5V)&o z-B|yyCs}lC=fKI0&SVyYj`lRTzi3ah;1D3^j|H|U#=w{-$HC!3V!_1y#sb#>jTclq zIe%a&d+(!1+hJ94tc| z1{?7?954mQ`QyOHpUF1Bbw)lSoYF|{Ibhe}-@|7uM;YrY4wL$@*+lu{fTARI6Nib{ z98MH;mT{OcS>*4rl#qxAf>;VNMp9aJ6i^C-8694{9rD6-F7B`)&2xv;lTIU2T~ zxp3x@dxlG6{-Z7%afh&@ab2KU$$17d0&x%EzM(Uo3*95vMtYaaR6<+}sC#t& za+&{q3Sokd1=9@K7F;G9*}GgOpyb%VMMQmu!;#dJT$VBi$r?P1iR*HdkgUOhgye;d z_bQhSMh)3Ncq0_88=Hmq3~(7}%{iDS@qe_(aOtD*>_ zQ*R#s?=^yQ|9yp=z+gd8IW|0AOV&j;A&|C+*Q2O#($@tHtYz}|7)aOA;Eh#M7XuTH zd>6owPeL)~q>cp)tP8STHiz`zXyBERdI%m^Mf)5e@X(3hgJPt8Kt0Y%+8cl&*#IXQ z@q{!QoU3G?Db-2NADAknj)1yg0h0X#ewVzz}Z>7&J_b7q$_}4?#ID+DCMl zEmW6AeX(QW9MCDk31L9gAPn&;G&;Dyem7ba1#xodcmKJTKr#k^2o+7s)A@%D7!n&!qg(!LcCD z0Llv>&mO@3`yR#wJxQrLa(%(FMfw2D1JZt`QG6^p4vN|&*B5ph8XH)Sq^$?h4-`8A z3~~B^A^!+4E@`VkI1j}KL7G`ecY?|$c@-#!d`SrVp|~kv;Or7I$pD&$lu3A@o8$>W zC4$5db-^4$^9(#3>N9)<6v;TKQzS3&1xEaPaF&sMhV6#VH{j-wP6Vryv{gXCDxvv< zPs^aQ2PB;26Df61##jJD{17;tpcKiu0wM`x3^MLQC1~Jzf&Git2=*_zx0G3k^QDd> zJf|!$JjwpCsJG20>P}@AhqPH~pu$K$4=}8olp)kX>KA1;^5dZ{hvY2)1M7_J3kMt( zgn>hiuz$z~&_1LtRtCQo{T`Tgs2prnQeT1BNb<2%xSQk@s3RWPAi#?vIR#!6*%nla z2swYii;_7G5RC%HgqQ=c^w9bOFN$m$U|C6jhB86WIRoxAS`W~$=*(l%zyl%sLdWMc zSZkzz4i+4f)p#E|HvA4J=CjovBpO-*EAw9$b^-St3$S6W*01M(N zjpTp@dV{phfcr-_5}Y2$wqOC>NvPEO~}P1dWVW&{!N`kx^X`UJ`?|5w#2MAet90SPiHy1h)~!K{ho+ ziqJX8R)PQ=Q5Q@QlJ|yGFnoSMYyyn~m}_+YLTm!b0mLSdE`gJlWC9@1gp3ZeM}?+v#BP7^5`;IX4M0+S!f6^JB~Q&79O4G6W7>kDxRJPvRo zNZA1M5BZIdrA6kj(AXTvwj$S;gKZE{?PP8()J3rvaL!SD3oz`%vVlrNa>Zp~S>tk$ zOmZNVgzPhHOLTs~Od*~SvKVn)Fr3K!0r3iSPE#Qpa-Km$pmpOg!5Agx6pk`9UYJs} z9&m!9y#QG)=#1w;AQAN$L=yFnjb#_oBFPvZq)KuT2L+KQv>w3KqjA8oig+Y0q;8S> z4a{~jogV7Q52fRGmN5IP?yBK(RGL8tR8yoq9z`-M48CLBd7_bTx`vR$) zs4sxgNgD((#AgGBa@7EX#6GeO;6aGJ540Y#sR09T4b??CsZb7*0?Bf4{-U}-nBa2A zUj+jMG(P%0E-44#n4;JnVCb9yiV@uzplCLU)2BSbSP93Yb9EvHy44`~W*a?WM0u1Hk z1I8rzM8Mdj9RV0b*2q4CaY4#B>s37`~yR|3-Ei8 zC`0}p*XIh2YZV{H_M)&sZ#(l!HP9YRgSS^`FL zC{Pahrhq}<0M!L)LzpriFC-w4IvXO{|G=Ln zWev0t$&o=_2I&g{hU|90;F1n%1DHaDp?Eyh#X1|@BeXAp{X_c_Ftjh>v?P5nIPg#m zh{Et10d<4M0pUh84zNPV^9}egyhd=hiro9ak>NE0EsNU#=9Zj4=ryTd;Aq7Ah|b3Q z2n-I~26%s{4M;R6G5DM>8V5rO?;~)#NxKO`UU(m2UXL=@knRKwuMxxz&>BHnE*cx; zo1wA6S&Q~BP&{b=f-6qySQc&m z^)7^1aJ$MR2TG+7AUhT^2*}tcl*2Y5V8}KD7LLrffV#l^pgDl7PlRE=5kjQM4~6J2 z$zy_f4_+PldvK8ewE^}i(tp4oU<@o?^n2*88|79Y--Wu(kK2XQ8~q+!EJ7HHM?o9F z&Y^PXP7Rbpyd#BSIfX+MwF`$2+6$oEQM(WjKx1Qp=R@`nHZX1jG8#}hh?XM^`?V~X zDzwibRF3vJd}AERJ&TQX3Apx1mp~{6xABiN8OU(d2Dme596(f~Heh+tTtQj^{yj*f zCFh0B!2M$@V_gTl0Gekuy2A$UFYI8{#y{qd&Bihg$?RyZAfFJeIq-?7FB}xh0w)jq zE)a>q{ezexYJ-FM9BO&d{87pA<|Wu17lbZK^h|-8_KhUvB5;6-$StpCFG2Hh_OYamVNyW>^^?)=y zq$dGG{wlDnNKZl*BeIJDLwb_B8${|!P*+G#0*3S?Xan-z9c(sm2#K`>PY7{o5Me?4 z2r#sdfRaIWCoqi2?gR|kouFco{{|T1$RL9S&jAe%Jv0t5_J`2!611^hvBypUajbOc~% z|58&x&L0Qflff9~8DQ1Od=Lm^;WK~(YfkDMc!>t-COGm)UxP})C2cOiP!2tHlZQN` z0E3&A6W1IZU0{s}4AMc!@dC+@jH-23qU_3y+PeH zBIAe*$aEofB0NCCByDeqlq-?ED20)71sIL=qX0v^0APrNqA+r<0E6VRiE}_=KB0K$Z<3j&RP53FTEcEJ+|!6)nDTtw&y znghTfseEExz`(k|80l9)Imj450g@Hvk^qM8!UKkKq5(skG$eUIfPq*ycuotgCEN%heLtuRhXAUJ@?N1Fx_b@O zD~j;}hT^N>fFa)nFy#LNhVHKd29H0H{e$NyP@h3gN!evkms`lXaH?Po`InIRf$pmT zhVs$DctLp-fFZvYFqGE<7`i76six@M0!*2-6JdA!qYXBCABm!sN&X%%bpHl?V?H67515o7CfA5u$hBz!is* z6YGL?faV@jYtdV4kPM9OX#$3F5g`Rv3AF)dBWYs-D~fngc>D$FShh0cvyy$LjzL0) zK}XR0r;sTQ7jlU4!cBP6*MM@6K}nXQA-@|KS|B*ca*)zN_65>UNj?lLE_7!ZvOB?m zA%71xH6go@@PyX`G7U%@2_Bw+>_+nUKphae9m;{SBg=uUhuQ^dj@Vmp(HGf&R0b7k zgRo(5k@E}?^6g=}AubItbcVtf1V(IPpTV6*dmj`8+#({%0SiakdXUYJ?ub$8z@%?a zy+TUzND#0=`J(Xp5bT@2~DBl|}^v)e%$c6(9TpH9a1jjH&#>=2C z;*0=8xwRBV#v38I1aTP@M*2N~!7W9yFL0m}dj`rO+Z!-2*vPstTtdd-@f;Mp0}RFA zsMnxKZXJ@KPz(()q=x`QwiNsrSixlfAn-@*Hz%0lk)-=7wL0SH@Zkp8ZhM3QLmMf zb`xM=EfeE~-AkV5P!8Rlqte^ay1{Wt`q)qo-IL=&Rye8K0fSc%$-cnCp*f%~jFaV{ zDk;xkJ(KnUcr)lNLb%k1-b?}vy~PR`%C`Uv<*5Spk3K^(9hzs*r)X?o@e;lob(@LQ zyMUqG3P=D#b`xOeJc6qQD90Z#tp6zc0QC>&1VRQMI;R1HV~SWK$f+b_1h5lOjv-(` zE}^<8R!K$LNc$9C--TCU$hz=M9GF3x|?Ba!O{{sMV+ zQ!fFNb{JqNmJb->G2vPVjg0XDhVrTaLp(KL=>1d3vP2v*V8~y93)S!dJ~^l0@uT$x zFO1xmaQzq0Gth~s9B?COjqp8ONJxO&Rb*eli9o*xA$rt5xL8E)LCB9Kc|u_M{~@O! z_k?W=<&d32MF>b67?J?slp^{J)H)e=fvY!=2uhX%{Y=h1l_!WXtPg+?BRN&bqDA+? z07Gwqzzq|0uLCf!F3A2-gapFy-Uq4x$u8tyk-2_w0RiQpQO|vmIbM)}i|rT4M2>aKO;}dVrza8MvvBI2pju_fr4^ZAXrcN^(FLI8q41wgsqSylyye36dDm z`=WpWg-W)8@5zDD2?PRJ4y<`3Ym_9Qa)_san*it@E?_9<7%(i8)V3t&4@@aCz5tdF z;tHsz{*Zow_&Lc@Ksl6`1{ll&F*fQsRdW8|Ib>uvap3(S^1P(N2IO2pMkUG1gEj!i zjBFPSO4KgsXY_kuV3BJ?kqW3T$TMmK4pCH&V)e$X+ z0z!KW(&otgU3f?ytaS4CAeV$Z-{86x$-}_~F?^oGu0;MNMWT@V92|ZmyWnb|eNH*R zs4rB+o9r_Cif~_nId!;1RUVl zpSa&B47}3`Ye`|G{X${ny#&Ch%gtn6`1Z;~ok*1f?}{u3I(wo_LOGzDP+uUBjq8%W zDb$6uNYpMI0TbngDo64MfPwvt+5mNr#s=~{ah|EVB!2}M74#y<3;YIZ1MwSFT{5Sa z!pQp{fKeCB$Tq+cz!;f#NYy2CIsgNM67>%r^g!zY{2yuq>})h%I6fxIDYOBH9{N4x zBSJa2FNx{`3xL`M4+OOfn-#4a>`YQNz}z70P8dH-0@*+6S$lF`;Eh@$z7KUN@}F1_ z2;C8WGWFOJD1`s7oU@~ujjg4l_}aDNikfy-_Tn%`aeZ+`eS3RnacGzNf2WOy<$qsx zgd$tSP1b^!bPzt@)iWt-8=sRwD<=HPXK&@tH-R_3Z}nU)+2P}># Date: Mon, 9 Nov 2026 20:35:27 -0800 Subject: [PATCH 39/74] X1 result (total mass beats REAP-50 by 0.184 KL), plan revision 2, revised priors; prune.py --skip-shard; x1b follow-up script --- .../2026-11-08-v41-reap-recovery-log.md | 23 + .../2026-11-09-keep-findings-and-levers.md | 130 +++++- .../2026-11-09-routed-precision-plan.md | 439 ++++++++---------- scripts/runpod/v41_reap/prune.py | 14 +- scripts/runpod/v41_reap/x1b_pod.sh | 20 + tests/test_runpod_v41_reap.py | 13 + 6 files changed, 394 insertions(+), 245 deletions(-) create mode 100755 scripts/runpod/v41_reap/x1b_pod.sh diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 75dc621..8efc10f 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -741,3 +741,26 @@ What it says: expert bytes read per token by 5%. - Item 4's 10 points need R to beat D as well, so the score waits for pass 2b. Missing static point: 120 experts at flat 2 bits (240 units), to compare with c48 directly. + +### 2026-11-09: X1 result, re-base decision, plan revision 2 + +- X1 finished in 1.6 h of GPU work (smoke 02:23 UTC, scores 03:59 UTC). Domain-mean KL on 54 eval + sessions: shipped REAP-50 0.716, one-pass total mass 0.532, prune-twice 0.546; at 96 experts the + Mac's pick 1.163 and prune-twice 1.154. Total mass against REAP-50: −0.184, session-level 95% + interval −0.277 to −0.128, better in 36 of 54 sessions; english-chat is worse (0.448 → 0.572). + Tables, intervals and the reading are in the + [findings note](2026-11-09-keep-findings-and-levers.md), section 5.1. Outputs: + `~/Models/v41-reap-run/x1/pod/` (scores, tables, per-position eval files, 2.4 GB). +- Follow-up on the same pod while it was hot (`x1b_pod.sh`): 96 experts in one pass on full-model + routing (`fm96`, keeps 74.0% of mass) against two passes through `fm192` (`pf96`, 81.2% of the + 192's mass). +- Jack approved re-basing on the total-mass 192 and asked for a disk cleanup. Export running on + the pod's CPU alongside the follow-up: `prune.py --skip-shard` (new, tested) leaves out the two + Engram shards, which are byte-identical to REAP-50's, so the export is 152 GB instead of 341. + Network volume grown 600 → 680 GB to hold it (volumes cannot shrink; $48 a month until deleted). +- Deleted on the Mac with Jack's confirmation, 373 GB: `~/Models/v41-reap75` (on Hugging Face), + `quant/e8p-prunedimp`, `sidecar/codes-prunedmass-96`, `sidecar/mac-4k-r256`, `sidecar/smoke`. +- Plan revised ([revision 2](2026-11-09-routed-precision-plan.md)): re-base first (WR), main + ladder 2-bit base + 1-bit residual, W4b redefined, sidecars and prune-twice closed, X3 demoted, + X8 (domain-balanced selection) added, operating rules for pods and evals. Five predictions are + registered in the findings note, section 6, before the runs that test them. diff --git a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md index 76a3359..fb42fbe 100644 --- a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md +++ b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md @@ -4,8 +4,10 @@ Written 2026-11-09 for the paper track (*Offload the Residual, Not the Expert*). whole project has measured that may not be in the literature, and whether any win is a bigger lever than it looked. Three sources: this campaign's own results ([recovery log](2026-11-08-v41-reap-recovery-log.md)), a survey of the older KEEP documents, and the -prior-art search in the [plan](2026-11-09-routed-precision-plan.md). The scheduled experiments (X1–X7) -are in section 7 of the plan; this file holds the evidence and the reasoning. +prior-art search in the [plan](2026-11-09-routed-precision-plan.md). The plan (revision 2) holds +the schedule and the status of each lever; this file holds the evidence and the reasoning. Sections +5 and 6 were added the same evening, after the first results: read them before sections 3 and 4, +several of whose bets they overturn. Nothing here has been checked against the literature beyond the six papers in the plan. Each lever gets its own search before it is claimed. @@ -120,7 +122,9 @@ case for lever X4. ## 3. What looks like it could go beyond the prior art -In the order I would bet on them. X numbers refer to the plan. +In the order I would bet on them on the morning of 2026-11-09. X numbers refer to the plan. +Outcomes so far: item 1 half right (the ranking rule, not the second pass), item 2 demoted, item 6 +promoted to the main line. See section 6. 1. **Prune twice, at the first cut (X1).** The one clearly significant win of this campaign has only been applied to the second halving. REAP-50's own KL (0.578) is the floor under every number @@ -174,3 +178,123 @@ what it pays is traffic, three units per promote/demote cycle against one, and a accurate high tier in return (0.090 against 0.101). The comparison is therefore run both ways, and at equal SSD rate as well as equal memory. The title's claim may turn out to be about traffic and simplicity more than about memory. + +## 5. Results of the evening of 2026-11-09 + +### 5.1 Selection at the first cut (X1, pod) + +Students of the full model, 54 eval sessions at full length, KL against the full model's logits. +Domain mean over 11 domains; pooled over positions in brackets. + +| Student | Experts | KL | Top-1 | +|---|---:|---:|---:| +| Shipped REAP-50 (`reap192`) | 192 | 0.716 (0.950) | 0.800 | +| Total routed mass, full-model routing, one pass (`fm192`) | 192 | 0.532 (0.650) | 0.814 | +| 384 → 288 → 192, routing re-measured on the 288 (`pt192`) | 192 | 0.546 (0.663) | 0.809 | +| The Mac's 96 inside REAP-50 (`pm96`) | 96 | 1.163 (1.443) | 0.722 | +| 384 → 288 → 192 → 96 (`pt96`) | 96 | 1.154 (1.268) | 0.703 | + +| Difference in domain-mean KL | Point | 95%, positions | 95%, sessions | Sessions better | +|---|---:|---|---|---:| +| `fm192` − `reap192` | −0.184 | −0.202 to −0.160 | −0.277 to −0.128 | 36 of 54 | +| `pt192` − `fm192` | +0.014 | +0.006 to +0.020 | +0.002 to +0.043 | 17 of 54 | +| `pt96` − `pm96` | −0.009 | −0.036 to +0.008 | −0.074 to +0.065 | 18 of 54 | + +- The ranking rule carries the gain. REAP's per-route score ignores how often an expert is used; + total routed mass counts it. `fm192` keeps 92.5% of the full model's routed mass and overlaps + REAP-50 by 76.5%. +- By domain the gain is largest where records are long and hard (agentic-python 1.339 → 0.897, + tool-use 1.995 → 1.294) and absent or negative where calibration data is thin: english-chat goes + from 0.448 to 0.572. Calibration tokens by domain run from 43K (business-email) and 52K + (english-chat) to 451K (tool-use). Total mass inherits that imbalance; REAP's per-route mean does + not. This is lever X8. +- Re-measuring routing on the 288-expert model did not help at 192. Confound: the two-pass tables + were ranked from 2 of 24 calibration sessions (257K tokens), the one-pass table from all 24. +- How this fits the Mac result (192 → 96 inside REAP-50: rule alone −0.025, pruned routing alone + −0.025, both −0.177): a cut that keeps 92.5% of mass barely changes routing, so the full + model's statistics describe the pruned model. A cut to 96 keeps about 74% and the routing it + leaves is different. Working rule: re-measure when the cut removes a large share of mass. The + 96-expert follow-up (5.5) tests it. + +### 5.2 Routed precision, pass 1 (Mac, inside REAP-50, 1-bit base + 1-bit residual) + +Numbers and intervals are in the recovery log. The decomposition at 288 units per layer: + +| Configuration | KL | Cost over 192 experts at flat 2 bits (0.747) | +|---|---:|---| +| 144 experts at flat 2 bits | 0.812 | +0.065: 48 experts dropped | +| Routed cache, 192 bases + 96 residuals, τ 0.2 | 0.832 | +0.053 nesting, +0.032 misses that fall to 1 bit | +| Fixed hot set, same codes | 0.905 | +0.053 nesting, +0.105 misses | + +- Routing the cache cuts the cost of misses by two thirds against a fixed hot set (0.105 → 0.032) + with 10% of slots still missing. The remaining miss cost is what a 1-bit fallback costs. +- The nesting penalty alone (0.053) is nearly the whole cost of dropping 48 experts (0.065). With + a 1-bit base, nested codes cannot beat pruning by much even with a perfect cache. +- SSD traffic was not binding: 0.28 GB/s at τ 0.3 and 0.83 GB/s at τ 0.2, against a 2.5 GB/s + ceiling. A separate-copy cache moving three times the bytes would still be under it. The + residual's traffic advantage only matters for small caches (K = 48 at τ 0.2 reads 2.72 GB/s). +- Per-token tiering with everything resident (MatMoE's setting) costs 0.022 KL for 54% fewer + tier-2 slots and saves no memory. Expert bytes are about 11% of decode traffic on this family, so + it buys little speed either. +- Per-token tier 0 is free at 7.7% of slots (+0.001 KL). + +### 5.3 Sidecars + +Closed. Fifteen fitted layers made KL worse by 0.199 [+0.164, +0.219] while every layer's holdout +objective improved. Details in the recovery log. + +### 5.4 What a unit buys + +From the static points inside REAP-50 (Mac eval), per unit per layer: + +| Step | KL change | Per unit | +|---|---:|---:| +| 192 experts at 1 bit → 96 experts at 2 bits (192 units either way) | 1.358 → 1.039 | bits beat breadth at equal memory | +| 96 → 144 experts at 2 bits (+96 units) | −0.227 | −0.0024 | +| 144 → 192 experts at 2 bits (+96 units) | −0.065 | −0.0007 | +| 192 experts, 2 bits → native FP4/BF16 (+384 units or more) | −0.169 | at most −0.0004 | + +The second bit is worth far more than the first expert it displaces; the last 48 experts are worth +little; and 0.17 KL sits above 2 bits. If a third bit recovers most of that 0.17 on the experts +that carry most of the routed mass, 96 residual units on the hot experts are worth more than the +96 units that buy experts 145–192. That is the case for the 2 + 1 ladder, and it is a guess until +the residual is fitted and scored. + +### 5.5 96 experts, one pass against two (pod follow-up) + +Pending when this section was written; results are appended to the recovery log. + +## 6. Priors revisited + +What I believed this morning, what the evidence says now, and what I do differently. + +| Prior | Now | Evidence | Consequence | +|---|---|---|---| +| "Prune twice" is the lever, and it will repeat at 384 → 192 | Half wrong. The ranking rule is the lever at the first cut; re-measuring matters only when routing changes a lot | 5.1 | Re-base on `fm192`; rule stated as "rank by total mass on routing that resembles the target" | +| REAP-50 is a sound floor for everything else | Wrong. It gives up 0.18 KL at the same size | 5.1 | WR in the plan; every later number is re-measured | +| A 1-bit base is weak but usable as a fallback tier | Wrong. It is the dominant cost: 1.358 alone, and each missed slot falls to it | 5.2 | Main ladder becomes 2-bit base + 1-bit residual | +| Nested codes cost "about 12%" and that is acceptable | The 12% in weight error is 0.053 KL end to end, as much as dropping a quarter of the experts | 5.2 | Nested codes must be judged end to end; a joint fit is in W7 | +| The residual wins on memory or on traffic | Memory: only against keep-both caches. Traffic: not binding at K = 96 | 5.2 | The title's claim is at risk; decided by the separate-copy sweep and the 2 + 1 ladder | +| Keeping all 384 experts at 1 bit could beat pruning half (X3) | Very unlikely | 5.2, 5.4 | Demoted; W4b no longer fits 384 experts | +| Sidecars recover a useful share of the layer gap | Wrong end to end | 5.3 | Closed; no layer-local metric accepted as evidence again | +| The eval set is 22 independent texts | 19 at 1,024 tokens; two domains share one prompt | recovery log | Eval set version 2 | +| A network volume is a disk | It is a single-stream 120 MB/s device with 3 ms seeks | recovery log | Copy to local disk first; rule in the plan | +| X1 needs 6.5–7.5 GPU hours | 1.6 h | pod logs | Estimates from measured throughput | + +Predictions registered before the runs that test them (2026-11-09, about 20:40 PST): + +1. Separate-copy cache, K = 96, swap accounting, against the routed nested cache at 288 units: the + copy wins on KL by 0.03–0.05 (about 0.78–0.80 against 0.832) and also beats 144 flat experts + (0.812). Its SSD rate stays under 2.5 GB/s at τ 0.3. +2. Oracle cache at K = 96 lands within 0.015 of the routed LRU at τ 0.2: the policy is near its + ceiling and the remaining loss is the base. +3. 144 experts all at nested 2 bits (288 units) scores about 0.86, worse than both the routed + cache (0.832) and flat 144 (0.812). +4. 96 experts in two passes through `fm192` beat one pass on full-model routing by at least 0.05 + domain-mean KL. +5. On the 2 + 1 ladder, 144 experts at 2 bits plus a routed cache of 96 third-bit residuals + (384 units) beats 192 experts at flat 2 bits by at least 0.03 KL. + +If 1 holds and 5 fails, the paper is about routing a precision cache and about selection, and the +title changes. If 5 holds, the title stands with the qualifier that the base must be a usable +model. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index dc57509..502369e 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -1,280 +1,239 @@ -# Routed precision: implementation plan to 100% - -Written 2026-11-09. Target headline (Jack, 2026-11-09, after the prior-art search below): *Offload -the Residual, Not the Expert: Unifying Pruning, Quantization and Offloading in MoE Inference*. It -replaces *Precision as a Routed Dimension*, whose first half MatMoE already covers. Progress is scored with the fixed rubric in the -[recovery log](2026-11-08-v41-reap-recovery-log.md) ("Headline progress rubric"); this plan says -what has to be built, run and measured for each rubric item, in what order, and what counts as -done. Score today: **26 / 100**. - -## 1. Prior art found today, and what it leaves us - -A search on 2026-11-09 found work that already covers parts of the idea. This changes what the -paper can claim, so it comes first. +# Routed precision: implementation plan to 100% (revision 2) + +First written 2026-11-09; revised the same evening after the first routed sweep, the X1 pod run +and the sidecar result. Revision 1 is in git history (`fd3d57e`). Target headline (Jack, +2026-11-09): *Offload the Residual, Not the Expert: Unifying Pruning, Quantization and Offloading in +MoE Inference*. Progress is scored with the fixed rubric in the +[recovery log](2026-11-08-v41-reap-recovery-log.md) ("Headline progress rubric"); the rubric and its +weights do not change with this revision. Score today: **31 / 100**. + +Evidence and reasoning for everything below: +[2026-11-09-keep-findings-and-levers.md](2026-11-09-keep-findings-and-levers.md), sections 5 and 6. + +## 1. What changed, in four lines + +1. **The base model moves.** Ranking experts by total routed mass beats the shipped REAP-50 by + 0.184 KL at 192 experts on the pod's eval. Every Mac number so far sits on REAP-50. All further + work runs on the new 192 (`fm-192`). +2. **The ladder moves up one bit.** The 1-bit base is not a usable model (KL 1.358 on its own), so a + cache miss is expensive, and nesting on top of it costs 0.053 KL. The main ladder becomes a + 2-bit base with a 1-bit residual (a third bit). The 1-bit + 1-bit ladder stays as the + low-memory regime, measured by the sweeps already queued, with no new work. +3. **Routing the cache is confirmed; "residual beats copy" is not.** A routed cache beats a fixed + hot set by 0.068 KL at equal memory and 0.28 GB/s of reads. SSD traffic was not the binding + constraint at that point, which removes the residual's main advantage over a separate copy + there. The separate-copy sweep (tonight) and the 2 + 1 ladder decide the title. +4. **Three levers are closed:** sidecars (+0.199 KL end to end), pruning twice (no gain over one + pass at 192, none at 96), and the tool-use NLL effect (an artifact). + +## 2. Prior art (unchanged from revision 1) | Paper | What it does | Overlap with us | |---|---|---| -| MatMoE (arXiv 2611.01704, 1 Nov 2026, ICCAD 2026) | Nested ("Matryoshka") integer codes per expert; picks each expert's bit-width at run time from its routing score; llama.cpp engines on CPU and GPU | Per-token precision from routing scores with nested codes. This is the phrase "precision as a routed dimension". All weights stay resident; no offload, no pruning. | -| DynaExq (arXiv 2511.15015) | Hot experts resident at high precision, low-precision fallback for the rest, promotions and demotions applied asynchronously so the forward pass never waits | Our no-stall cache with async admission. Separate copies per precision, not nested. GPU memory, not SSD. | -| DyMoE (arXiv 2603.19172) | Run-time mixed precision by expert importance, depth-adaptive schedule, look-ahead prefetch | Depth-dependent importance (our "two routing regimes" observation), prefetch. | -| HOBBIT (arXiv 2411.01433) | On a cache miss, run a low-precision copy of a less critical expert instead of waiting for the load | Low tier as the miss path. Separate copies. | -| MC-MoE (arXiv 2410.06270) | Static mixed-precision per expert plus online per-token expert pruning | Per-token tier 0, but with static precision. | +| MatMoE (arXiv 2611.01704, ICCAD 2026) | Nested integer codes per expert; bit-width per expert at run time from its routing score | Per-token precision with nested codes. All weights resident; no offload, no pruning. | +| DynaExq (arXiv 2511.15015) | Hot experts resident at high precision, low-precision fallback, asynchronous promotion | Our no-stall cache. Separate copies. GPU memory, not SSD. | +| DyMoE (arXiv 2603.19172) | Run-time mixed precision by importance, depth-adaptive, look-ahead prefetch | Depth regimes, prefetch. | +| HOBBIT (arXiv 2411.01433) | On a miss, run a low-precision copy instead of waiting | Low tier as the miss path. Separate copies. | +| MC-MoE (arXiv 2410.06270) | Static mixed precision plus online per-token expert pruning | Per-token tier 0 with static precision. | | PMQ, GEMQ (ICML 2026), EdgeMoE | Static per-expert bit allocation | Our step 2c allocator. | -Only abstracts were read for all but MatMoE (introduction and method overview). Step W0 below reads -them in full. - -What none of them appears to do, and what the paper therefore has to stand on: - -1. **The offloaded unit is the nested residual.** Tiers span unified memory and SSD inside one - code: the base is always resident, and a promotion reads only the residual (4.6 MB), never a - second full copy. What that buys depends on how a separate-copy cache (DynaExq, HOBBIT) is run, - and both readings are measured (corrected 2026-11-09; the first draft claimed only the first): - - if it keeps the low copy of a hot expert, a hot expert costs 2 extra units and a nested design - holds twice as many hot experts at equal memory; - - if it drops the low copy and reads it back on demotion, memory is equal, but every - promote/demote cycle moves 3 units (2 in, 1 back) where a residual moves 1, and a demotion is - free. At an equal SSD rate the nested cache can admit three times as often. - The separate 2-bit copy is slightly more accurate than base + residual (error 0.090 against - 0.101), so the nested design has to win on memory or on traffic, not on per-expert accuracy. -2. **Tier 0 in the same policy.** Expert pruning (static tier 0), per-token slot dropping, precision - and residency all come from one rule with one trade-off parameter. -3. **The sub-2-bit lattice regime** (1 and 2 bits per weight, E8 codes), on a 384-expert, top-6 - frontier model, on a 128 GB laptop. - -The new title leads with claim 1 and keeps "unifying pruning, quantization and offloading", the -half of the original headline that survives. The paper must cite and beat -MatMoE-style and DynaExq-style policies rather than only static baselines. If W0 finds a paper that -already offloads nested residuals under per-token tiers, the paper becomes an empirical study of -that design at 2 bits and below, and Jack decides whether to continue. - -## 2. Where things stand - -Done: per-token tiers in `local_eval.py` (`--dynamic`, `--tau`, `--tau-drop`), the residual cache -simulation (`residual_cache`, `--cache-k`, `--cache-warm`), the cost report (`routed_report.py`), -unit tests, and the static frontier to beat (192 units 1.014, 288 units 0.809, 384 units 0.747 KL). - -Running on the Mac (detached, scripts in `~/Models/v41-reap-run/jobs/`): the nested residual fit -(`quant/e8-nest-r1`, 8 of 40 layers), the step-3 sidecar fit (restarted from layer 0), and a chain -that runs the sidecar eval and then the first routed sweep (8 variants) when the fits finish. - -One unit = one expert at 1 bit per weight = 4.6 MB per layer. All Mac results are inside REAP-50 -(192 experts per layer), scored by KL against the full model's logits on 22 held-out sessions. +Only abstracts were read for all but MatMoE. W0 reads them in full. + +Claims the paper could stand on, with tonight's evidence against each: + +| Claim | Status | +|---|---| +| 1. The offloaded unit is the nested residual, not a second copy | Open and at risk. Needs R to beat D; with a 1-bit base the nesting penalty (0.053) exceeds the gap to plain pruning (0.025). Retested on the 2 + 1 ladder, where the base is usable. | +| 2. Tier 0 in the same policy | Supported in part: dropping the 7.7% of slots with gate below 0.1 costs 0.001 KL. No joint policy yet. | +| 3. The regime at 2 bits and below, 384-expert top-6 model, 128 GB laptop | Narrowed: below 2 bits the base is unusable. The regime is 2 to 3 bits. | +| 4. (new) Select experts by total routed mass on routing that resembles the pruned model | Supported: −0.184 KL at 384 → 192 (pod), −0.177 at 192 → 96 (Mac). Not yet searched in the literature. | ## 3. Workstreams -Each workstream lists the deliverable, how it is built, the acceptance test, and the fallback. -Times are estimates; the Mac runs one heavy CPU job and one GPU fit at a time, under the 80 GB cap. +Units: one expert at 1 bit per weight = 1 unit = 4.6 MB per layer. A 2-bit expert is 2 units, a +third-bit residual 1 unit, a native FP4 expert 4 units. Budgets are per layer. + +### WR. Re-base on the total-mass 192 (new; gates W4 onward; no rubric points) + +1. Export `fm-192` on the pod (`prune.py --skip-shard` for the two Engram shards, 152 GB). Running. +2. Download through a CPU pod on the volume, 16 parallel streams. On the Mac, clone the two Engram + shards from `~/Models/v41-reap50` (`cp -c`, no extra space) and check every file against the + export's `SHA256SUMS` and REAP-50's. +3. `local_eval.py` against the new checkpoint in BF16/FP4 on the Mac eval set. **Accept** when it + beats REAP-50 on the same sessions by at least 0.10 KL (the pod saw 0.184 on 54 full sessions). +4. Capture routing and importance on the new model (the step-2c capture, four chunks). +5. Codes: flat 2-bit E8P for all 192 (reuse the fits of the experts shared with REAP-50, 76.5%, if + the importance source is the same; otherwise refit all, about 7 h of GPU), then the 1-bit + residual of the 2-bit base (`--residual-of`, about 4.5 h). +6. Nested expert sets inside the new 192 by total mass on its own routing: 176, 160, 144, 128, 112. +7. Eval set, version 2: 4,096 tokens per record, and the four sessions that share a first 1,024 + tokens no longer counted as four texts. Sub-1-bit comparisons keep the old set for continuity. + +Only after step 3 passes is `~/Models/v41-reap50` removable (341 GB); the 1 + 1 sweeps must finish +first. ### W0. Related work, read in full (gates the claims; no rubric points) -- Read MatMoE, DynaExq, DyMoE, HOBBIT, MC-MoE and GEMQ end to end. Write - `2026-11-09-routed-precision-related-work.md`: one row per paper with nested or not, per-token or - not, what is offloaded, pruning, bit range, models, hardware. -- Done when the three claims in section 1 are each confirmed or struck. Half a day, in parallel - with the fits. +Unchanged. Adds a search for claim 4 (frequency-weighted or mass-based expert pruning criteria, +and calibration on the pruned model's routing). -### W1. Nested codes validated (rubric item 1: 5 → 10) +### W1. Nested codes validated (rubric item 1: done, 10) -- Finish `quant/e8-nest-r1` (32 layers left, about 6.6 minutes each). -- Accept when (a) every layer's base + residual error is within 1.15× of the flat 2-bit fit, and - (b) the sweep's endpoints land where they should: all slots at tier 2 scores KL ≤ 0.80 (flat - 2-bit is 0.747), all at tier 1 reproduces flat 1-bit (1.358). -- If (b) fails: one alternating round (refit the base against `W − residual`, then the residual - again), about 7 h of GPU per round. -- Stretch, for the frontier figure: a third rung (`--residual-of` taking two directories), about - 3 bits per weight, 4.4 h and 34 GB of disk. +Fit complete, 40 of 40 layers; base + residual error 0.102 / 0.084 against 0.090 flat; end to end ++0.053 KL [+0.039, +0.071]. The acceptance bound (all-tier-2 KL at most 0.80) was met at 0.8001. +The alternating refit is no longer planned for the 1 + 1 ladder; it is used on the 2 + 1 ladder +only if its nesting penalty exceeds 0.02 KL. ### W4. Core result (rubric item 4: 0 → 25) -The claim: at equal resident memory, routed tiers with residual offload give lower KL than any -static assignment and than a separate-copy cache, at an SSD read rate the Mac sustains. +The claim: at equal resident memory, a routed cache of third-bit residuals over a 2-bit base gives +lower KL than any static assignment and than a separate-copy cache, at an SSD read rate the Mac +sustains. -Budgets: B = 240, 288 and 336 units per layer (44, 53 and 62 GB of routed experts). A configuration -keeps N expert bases resident and K residual slots, N + K = B. Policies, all in the same harness on -the same codes: +Budgets B = 288, 336 and 384 units (53, 62, 71 GB of routed experts). A configuration keeps N +experts at 2 bits and K residual slots, 2N + K = B. -| Policy | What it is | How it runs | -|---|---|---| -| S, static-nested | hot set = top K by pruned-model routed mass, never changes | `--cache-k K --tau 1e9` with `--cache-warm` | -| R, routed-nested (ours) | LRU of K residuals, async admission at gate ≥ τ | `--cache-k K --tau τ` | -| D, separate-copy cache (DynaExq / HOBBIT style) | high tier is a second flat 2-bit copy. Scored two ways from one run: keeping the low copy (N + 2K units, 2 units per fetch) or swapping it (N + K units, 3 units per cycle) | `--dynamic BASE\|HI`; `routed_report.py` prints both | -| M, all-resident routed (MatMoE style) | every residual resident, tier by routing score | `--tau τ`, no cache; a latency baseline at 384 units | -| O, per-session oracle | hot set = top K by that record's own routed mass; upper bound for any cache | new: `--cache-policy oracle` | - -Build first (half a day): the separate-copy mode, the oracle policy, expert masks combined with -`--dynamic` (N < 192), a paired bootstrap over sessions in `routed_report.py --compare A B`, and an -admission trace output for W6. Move the routed forward and `residual_cache` into a shared -`routed.py` so the pod driver can use them. - -Runs (each pass is 8 variants, about 3.5 h on the CPU): - -1. Pass 1 (chained): N = 192; endpoints, S(96), R(96, τ 0.2 and 0.3), R(48, τ 0.2), M(τ 0.22), - per-token drop. -2. Pass 2 (queued): O(96), R(96, τ 0.15), N = 144 all at tier 2, N = 160 with K = 128 routed and - static, S(48), and the two projection-level rungs (X5a). -3. Pass 2b (queued), separate copies: K = 96 at τ 0.2, 0.3 and 0.4 and static; K = 48 and 144. -4. Pass 3 (queued), budgets 240 and 336: R, S and O at K = 144; R(48, τ 0.3), O(48); N = 160 with - K = 80 routed and static. -5. Pass 4: the winners and their baselines at 4,096 tokens per record (the pod's teacher logits - cover whole records), so the cache is measured past its warm-up. - -Acceptance and scoring inside the item: - -- 10 points: R beats S by at least 0.03 KL, and beats D at equal memory and equal SSD rate (under - whichever of D's two accountings is better for D), bootstrap 95% interval excluding zero, at one - budget, with simulated SSD traffic at or below 2.5 GB/s at 30 tok/s (half the measured sequential - rate). -- 15 points: the same at two of three budgets, confirmed at 4,096 tokens. -- 25 points: reproduced on the full 384-expert model on a pod (W4b). - -W4b, pod (only after 15 points locally): fit nested codes for all 384 experts and run the same -policies with the full eval set (108 sessions). Needs a torch/CUDA backend for the 1-bit E8 search -(256 codewords, a plain argmin) and the routed forward in `collect.py`. All prep on the Mac; the -code pack uploads while the model downloads. Estimate 6–10 h on 1×H100 at $3.99/h, $25–40 plus -storage; H200 at $5.29/h if H100 stock is still out. Budget left is about $380 of $497. - -If R does not beat S, in this order: - -1. Admission by prediction instead of after use: score layer l+1's gate on layer l's output and - fetch ahead (the Fate result, 97% hit rate). Measure the hit rate offline first. -2. A stronger ladder: 2-bit base plus 1-bit residual (3 bits), budgets 384–576. The 1-bit base is - weak (error 0.31); with a 2-bit base the question becomes how close to FP4 (0.578) the routed - residual gets. -3. A long-horizon hot set (DynaExq's rule) over nested residuals, which isolates the nesting gain - from the policy. -4. If all three fail, the honest paper is the negative result plus the selection finding from - step 2c, under a different title. Say so then; do not stretch the claim. +| Policy | What it is | +|---|---| +| F, flat | B / 2 experts at 2 bits, nothing else. The frontier to beat. | +| S, static-nested | fixed hot set of K residuals by routed mass | +| R, routed-nested (ours) | LRU of K residuals, asynchronous admission at gate ≥ τ | +| D, separate-copy | the hot expert's high tier is its native FP4 copy (4 units, swapped for the 2-bit copy: 2 extra units resident, 6 units moved per promote/demote cycle against 1) | +| O, per-session oracle | hot set = that record's own top K; the ceiling for any cache | + +Build: a base-unit field in `routed_report.py` (bases are no longer 1 unit); a native-precision +high tier for `--dynamic BASE|HI`; `--tau-file` (W5). + +Runs, on `fm-192`, eval set version 2: + +1. Pass A, B = 384: F(192); R, S, O at N = 160, K = 64 and N = 144, K = 96; endpoints (all at + 3 bits, all at 2 bits). +2. Pass B, B = 288 and 336: F(144), F(168); R and S at N = 128, K = 32 and 80; N = 112, K = 64. +3. Pass C: D at the K and τ of the best R in each budget; tier 0 on top of the best R. +4. Carried over from revision 1 and already running on REAP-50 (1 + 1 ladder): pass 2 (oracle, + N < 192, projection rungs), pass 2b (separate copies), pass 3 (budgets 240 and 336). + +Acceptance and scoring inside the item (unchanged in form): + +- 10 points: R beats F and S by at least 0.03 KL and beats D at equal memory and equal SSD rate, + session-level bootstrap 95% interval excluding zero, at one budget, with SSD traffic at or + below 2.5 GB/s at 30 tok/s. +- 15 points: the same at two of three budgets on eval set version 2. +- 25 points: reproduced on the pod's eval (108 sessions, full length) with the routed forward on + GPU (W4b). + +W4b is redefined: it no longer fits codes for 384 experts (X3 is demoted, see section 5). It ports +the routed forward to `collect.py`, uploads the Mac's codes, and scores the winning configurations +on the pod. Estimate from tonight's throughput (about 11 minutes per stream over 54 sessions): +2–3 h on one H100, under $15. + +If R does not beat F on the 2 + 1 ladder either: + +1. Admission by prediction (score layer l + 1's gate on layer l's output), measured offline first. +2. Report the routed-cache result against S and D as the contribution, with the selection finding, + under a title that does not claim to beat pruning. Say so then; do not stretch the claim. ### W5. Unification (rubric item 5: 0 → 10) -- One rule for all three decisions. For a slot with gate weight g on expert e in layer l, the - estimated benefit of tier t over t − 1 is g² · (ε_{t−1} − ε_t) · ‖f_e‖², with ε from the fit - metrics (ε_0 = 1) and ‖f_e‖ from the routing capture; the cost is the bytes that tier adds - (resident or read). A slot takes the highest tier whose benefit per byte is at least λ. One λ - yields per-(layer, expert) thresholds for dropping and for the residual, and the same test on - expected benefit decides which bases and residuals are resident. New `routed_policy.py` writes - the thresholds; `local_eval.py --tau-file` reads them. -- `routed_frontier.py`: KL against resident GB for static REAP + KEEP, static-nested, separate-copy - and routed, with SSD rate marked; figure and table committed. -- If the sidecar eval shows at least 0.05 KL, add sidecars as an always-resident layer tier (about - 1.1 units per layer). -- Accept when at least 12 measured configurations are on the figure and the one-λ policy is within - 0.02 KL of the best hand-tuned setting at each budget. +Unchanged in design (one benefit-per-byte rule with one λ; `routed_policy.py`, +`routed_frontier.py`). Changes: tier 0 enters with tonight's measurement as its first point; the +sidecar tier is removed; the frontier figure plots both ladders, with F, S, R, D and O. ### W6. Real Apple Silicon streaming (rubric item 6: 0 → 10) -Trace-driven, since no V4.1 decoder exists on this Mac. - -- `residual_cache` writes its admission trace (record, token, layer, expert). -- `residual_stream_bench.py` replays the trace at 10, 30 and 60 tok/s against the real residual - bytes on the internal SSD: `pread` with `F_NOCACHE` into page-aligned buffers from a small thread - pool, reading each expert's extent straight out of the uncompressed npz (no second copy on - disk). Also an `mmap` + `madvise` variant. -- Measures: latency per admission, the share that lands before the next token and before that - expert is next routed, sustained GB/s, CPU use, `phys_footprint`, thermal state, bytes read per - 1,000 tokens, and the copy cost into an MLX array. -- Accept when, at the winning W4 configuration, at least 99% of admissions complete within one - token interval at 30 tok/s and the sustained rate is at most half the device's sequential rate. - Run with no other heavy job, because the fits and evals also read from this SSD. +Unchanged in design. Two notes from tonight: the admission rates to replay are low (9.5 to 28 MB +per token at K = 96), so the bar is latency per admission, not sustained throughput; and the +network-volume episode is the same failure in miniature, a single reader faulting pages one at a +time, so the bench measures `pread` from a thread pool against `mmap` faults (X4) on purpose. ### W7. Learned policy and tier-aware codes (rubric item 7: 0 → 10) -- (a) 5 points: the calibrated per-(layer, expert) thresholds from W5 beat a single global τ by at - least 0.02 KL at equal memory and SSD rate. -- (b) 5 points: tier-aware codes. Capture the importance diagonal separately for low-gate and - high-gate slots, fit the base to the low-gate statistics and the residual to the high-gate ones - (each code is fit to the tokens that use it), and gain at least 0.02 KL. Four capture chunks, - about 7 h of GPU for the refit, one eval pass. -- Not required for 100%: end-to-end distillation of thresholds and router biases with a bits - penalty. It needs backprop through the streamed model on a pod (over $100) and is worth doing - only if (a) and (b) leave a visible gap. +- (a) 5 points: per-(layer, expert) thresholds from W5 beat one global τ by at least 0.02 KL. +- (b) 5 points: a jointly fitted nested code (base chosen so that base + residual is best, not the + base alone) or tier-aware importance, gaining at least 0.02 KL at equal units. ### W8. Generality (rubric item 8: 0 → 10) -- 5 points, second MoE: Qwen3-30B-A3B (specs to verify at download: 48 layers, 128 experts, top 8, - Apache-2.0, about 61 GB in BF16). It fits the Mac for the teacher pass and, unlike V4.1, can - generate locally. Build its pack with the same domains, capture routing and teacher logits, fit - nested E8 codes, port the routed forward to its MoE module, and repeat the S / R / D comparison - at three budgets. -- 5 points, downstream: on Qwen3-30B, two generative benchmarks inside the scope (coding and tool - use), chosen after a currency check so that nothing saturated is used, at least 200 items each, - static against routed at equal memory. On V4.1, likelihood-ranked tasks that fit the streamed - teacher-forced pass (ranking the correct patch or tool call among distractors) built from - held-out sources. -- Needs about 75 GB of disk and Jack's go for the download. +Unchanged (Qwen3-30B-A3B, download needs Jack's go). Adds the selection rule as a second thing to +replicate there, since it needs only routing statistics. ### W9. Paper (rubric item 9: 1 → 5) -- `docs/paper/` in arXiv LaTeX: method (tiers, nested lattice codes, the one-λ rule, the no-stall - cache), results (frontier, SSD traffic against τ and K, depth regimes, per-domain), related-work - table from W0, limitations (teacher-forced KL, one eval length, the tool-use anomaly). -- Sections are drafted as each workstream lands; the full draft follows W8. Publishing the nested - codes on Hugging Face needs Jack's OK. +Unchanged, plus a selection section (claim 4) that can also stand alone as a short note. + +### WS. Selection, to its own conclusion (new; supports W9 and a possible REAP-50 v2) + +- Tonight: 96 experts in one pass on full-model routing against two passes through `fm-192` + (`x1b_pod.sh`), to test the rule "re-measure routing when the cut removes a large share of mass". +- X8, domain-balanced mass: calibration tokens are uneven by domain (english-chat 52K, tool-use + 451K) and english-chat is the one domain that got worse (0.448 → 0.572). Needs per-domain + accumulation in `collect.py` and one calibration pass on a pod; run with W4b. Adopt only if it + gains at least 0.03 domain-mean KL, because adopting it means re-basing again. +- Publishing a replacement REAP-50 on Hugging Face is Jack's decision. ## 4. Order and expected score | When (estimate) | Work | Score after | |---|---|---:| -| Today | W1 finishes; sidecar eval; sweep pass 1; W0 reading; W4 harness additions | 31 | -| Day 1 | W4 passes 2–3; W6 bench built and run between passes | 41–46 | -| Day 2 | W4 pass 4 (4,096 tokens); W5 policy and figure; W7a | 61–66 | -| Day 3 | W6 final numbers; W7b capture, refit, eval; W4b prep | 76 | -| Day 4 | W4b on a pod; W8 download and teacher pass | 86 | -| Days 5–6 | W8 fit, routed eval, downstream benchmarks | 96 | +| Tonight | 1 + 1 passes 2, 2b, 3 on REAP-50; X1 follow-up; export and download of `fm-192` | 31 | +| Day 1 | WR steps 3–7 (capture, fits); W0 reading; W6 bench built on tonight's traces | 31 | +| Day 2 | W4 passes A–C on `fm-192`; title decision | 41–46 | +| Day 3 | W5 policy and figure; W6 numbers; W7a | 61–66 | +| Day 4 | W4b and X8 on a pod; W7b | 76–81 | +| Days 5–6 | W8 | 91–96 | | Day 7 | W9 full draft | 100 | -The score moves only on measured results, so a failed acceptance test holds it where it is. - -## 5. Resources - -- Memory: 80 GB cap. The 8-variant routed sweep is estimated at about 65 GB; if the guard trips it - splits into two passes of four. -- Disk: 74 GB free after storing 1-bit codes as uint8. The nested fit takes 27 GB more. W8 does not - fit without freeing space. -- Pod: about $380 left. W4b is the only planned spend ($25–40). - -## 6. Decisions for Jack - -1. Disk: Jack is cleaning up himself (2026-11-09). Safe to remove: `~/Models/v41-reap75` (273 GB; - published on Hugging Face, and its experts are a subset of the local REAP-50) and - `~/Models/v41-reap-run/quant/e8p-prunedimp` (65 GB; step 2 found it changes nothing). In use, do - not remove: `~/Models/v41-reap50` and, under `~/Models/v41-reap-run/`, `stats`, `pack`, - `imatrix`, `alloc`, `evals`, `sidecar`, `jobs`, `quant/e8p-podimp`, `quant/e8-1bit-podimp` and - `quant/e8-nest-r1`. -2. Download Qwen3-30B-A3B for W8. -3. Whether the prior art in section 1 changes the title or the ambition. - -## 7. Candidate levers beyond the prior art (2026-11-09) - -From this campaign's results and a survey of the older KEEP documents. None is checked against the -literature yet; each gets a search before it is claimed. Ordered by expected value. - -| # | Lever | Evidence in hand | Test | Where it runs | -|---|---|---|---|---| -| X1 | Prune twice at 384 → 192. REAP-50 was chosen on full-model routing with the per-route score; its KL (0.578) is the floor under everything here. | The same fix cut KL 15% at 192 → 96 (step 2c ablation); a second pass saturates (step 2d). | Calibration pass on the full model masked to 288 by total mass, re-rank to 192, eval against shipped REAP-50. | Pod, with W4b; about $15–25 extra | -| X2 | ~~Calibration-targeted pruning as specialization.~~ Dropped 2026-11-09. | The NLL advantage sits only where the teacher is surprised by off-format text embedded in the records; on teacher-confident positions the student is worse in every domain. | Done locally from the pod's stored outputs; see the findings note, section 3. | — | -| X3 | Keep every expert, offload residuals, instead of pruning half. | V4.1 plan: each expert kept costs 0.359 GB of RAM at 2 bits "but adds no per-token reads", and the fit "forces a ~50% prune". All 384 one-bit bases cost the same memory as REAP-50 at 2 bits. | 384 bases + K residuals against REAP-50 at 2 bits (0.747) at equal resident memory. This is the title's claim at full scale. | Pod, W4b; the Mac sweep at 192 vs 96 is its proxy | -| X4 | The OS page cache as the precision cache. Residuals are memory-mapped clean file pages; `mincore` asks whether one is resident, `madvise(WILLNEED)` requests it in the background, and the kernel evicts under pressure at no write cost. Precision degrades instead of the machine swapping. | Roadmap, 2026-08-21: at 90.99 GB of MLX-active memory byte-identical work spanned 16.5× in time and the bind alone logged 535,792 swapouts. Anonymous memory at that footprint thrashes; clean file pages cannot. | In W6: replay the trace through `mmap` + `mincore`, with and without a memory hog, and measure hits, latency and swapouts. | Mac | -| X5 | A stronger base and finer rungs. The 1-bit base is the weak link. | Pilot: 1-bit codes gave a 0.56 block cosine; the SwiGLU amplifies projection error about 2.8×; 2 bits is "at its rate-distortion limit". Here, a static 1-bit tier added 0.025 KL at best. | (a) Projection-level tiers from the existing codes: residual for the down projection only (1.33 bits) or gate/up only (1.67 bits), no new fit. (b) The 2-bit base + 1-bit residual ladder, moved up from fallback to W4 pass 3. | Mac | -| X6 | Budget by sensitivity, not routed mass. | Step 1: mass-based reallocation lost to uniform (1.046 vs 0.955 at 128) although layers 20–39 are far more concentrated than 0–19. | Mask a fixed share of experts in one layer band at a time and measure KL; allocate by measured sensitivity. Two eval passes. | Mac, after W4 | -| X7 | A shared base across similar experts (base = one matrix per cluster, residual per expert). | None; speculative. | Functional similarity of experts on shared inputs for 3 layers; stop if experts are near-orthogonal. | Mac, about 1 h | - -Smaller findings worth a paragraph in the paper or a note of their own: - -- Expert precision is a memory lever, not a speed lever, on this family: on V4-Flash, routed experts - were 11% of the 15.27 GB read per decoded token and attention 67% (roadmap, 2026-08-21). MatMoE's - latency argument for dynamic precision does not carry over; the memory argument does. The - `tok_s_bound` column of `routed_report.py` counts expert bytes only and is not a speed estimate. -- Incoherence processing (the random rotation QuIP# relies on) made importance-aware E8P fits 15% - worse on the down projection, because it flattens a per-column importance that spans 180× - (wave-5 pilot). -- A gate/up-swapped artifact loads, gives finite logits and scores cosine 0.9993; only a - discrimination ratio caught it. Cosine is not a correctness check. -- Negative priors for the old recovery steps 4–5: on GLM-4.5-Air router distillation was rejected - (top-1 change slightly negative) and a rank-4 sidecar improved top-1 from 0.795 to 0.852 but ran - 1.72× slower and failed the speed gate. Sidecars here are judged on quality per unit of memory and - on their per-token cost, and router distillation stays off the critical path. -- A quality reference from outside: the Rapid-MLX V4.1 build is reported at KL 0.26 and top-1 85% on - a 256 GB M3 Ultra (local test plan, 2026-11-08). Our best 2-bit REAP-50 number is 0.747. A result - that impresses has to close most of that gap at about half the memory. +The score moves only on measured results, so a failed acceptance test holds it where it is. The +re-base costs about a day against revision 1's schedule and raises every number it touches. + +## 5. Levers: status after tonight + +| # | Lever | Status | +|---|---|---| +| X1 | Prune twice at 384 → 192 | Resolved. The ranking rule is the gain (−0.184 KL); the second pass is not (+0.014 at 192, −0.009 n.s. at 96). The second pass used 12× less calibration data than the one-pass table, so "slightly worse" is not separable from sample size. | +| X2 | Pruning as specialization | Closed (artifact). | +| X3 | Keep all 384 bases at 1 bit, offload residuals | Demoted. A 1-bit base scores 1.358 alone and misses fall to it. At 2 bits, 384 bases are 768 units, over any budget here. | +| X4 | OS page cache as the precision cache | Open, in W6. | +| X5 | Stronger base, finer rungs | (b) promoted: the 2 + 1 ladder is now W4's main line. (a) projection rungs are in pass 2. | +| X6 | Budgets by sensitivity | Open, after W4. | +| X7 | Shared base across experts | Open, low priority. | +| X8 | Domain-balanced selection | New, in WS. | +| — | Sidecars (old step 3) | Closed: +0.199 KL [+0.164, +0.219] with 15 layers fitted. | + +## 6. Operating rules learned tonight + +Pods: +- Never run the model from a network volume: one stream reads 120 MB/s, a random row takes + 2–3 ms. Create the GPU pod with a container disk that holds the checkpoint and copy first + (16 streams, 476 GB in 7 minutes). +- Smoke-test every launcher on the image before the GPU clock matters; match readiness by + substring; `PIP_BREAK_SYSTEM_PACKAGES=1` on Ubuntu 24.04 images. +- Estimates from measured throughput: about 390 tok/s for a calibration stream and 11 minutes per + eval stream over 54 sessions. The X1 estimate (6.5–7.5 h) was 4× too high. +- CPU-only work (exports, transfers) runs on a CPU pod or alongside a GPU job that is already + running, never on an idle GPU. + +Evaluation: +- Report session-level bootstrap intervals; position-level intervals are about half as wide. +- Domain mean and pooled KL differ a lot (0.716 against 0.950 for REAP-50) because the agentic and + tool-use domains are long and hard. State which one a number is. +- Mac numbers (22 sessions, first 1,024 tokens) and pod numbers (54 sessions, full length) are not + comparable in absolute terms; only differences within one set are. +- A layer-local metric is not evidence. Twice now (GLM-4.5-Air, V4.1) a sidecar that improved its + layer objective failed end to end. Score a partial fit end to end before finishing it. +- Settle the expert set before fitting codes to it. +- Write the prediction down before the run (findings note, section 6). + +## 7. Resources + +- Memory: 80 GB cap; the 8-variant sweep peaked at 64.1 GB. +- Disk: 373 GB freed on 2026-11-09 (REAP-75, the no-effect refit, sidecar files). The new + checkpoint adds 152 GB; its codes about 100 GB. +- Pod: about $20 spent tonight (H100 about 4 h, CPU pod, volume). Volume `yqazs7aqe6` is 680 GB + ($48 a month) and holds the full model and the export; delete it after W4b. + +## 8. Decisions for Jack + +1. Title: decide after the separate-copy sweep and the first 2 + 1 pass. Until then it stands. +2. A replacement REAP-50 on Hugging Face built from the total-mass table (public change). +3. Qwen3-30B-A3B download for W8. diff --git a/scripts/runpod/v41_reap/prune.py b/scripts/runpod/v41_reap/prune.py index 61ca305..ab55b14 100644 --- a/scripts/runpod/v41_reap/prune.py +++ b/scripts/runpod/v41_reap/prune.py @@ -135,6 +135,8 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--survivors", required=True, type=Path, help="reap-select output for the backbone layers") parser.add_argument("--mtp-survivors", type=Path, help="optional survivor table for the mtp.* drafter") parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--skip-shard", action="append", default=[], + help="a shard the plan leaves untouched (the Engram tables): indexed but not written; use the source's file") args = parser.parse_args(argv) index = json.loads((args.src / INDEX).read_text()) @@ -155,9 +157,17 @@ def main(argv: list[str] | None = None) -> int: new_map: dict[str, str] = {} total = 0 for shard in sorted(set(weight_map.values())): - names = prune_shard(args.src / shard, args.out / shard, plan) + if shard in args.skip_shard: + with (args.src / shard).open("rb") as handle: + names = [n for n in read_header(handle)[0] if n != "__metadata__"] + touched = set(names) & (set(plan["rename"]) | set(plan["slice_rows"]) | set(plan["drop"])) + if touched: + raise ValueError(f"{shard} cannot be skipped: the plan changes {len(touched)} of its tensors") + total += (args.src / shard).stat().st_size + else: + names = prune_shard(args.src / shard, args.out / shard, plan) + total += (args.out / shard).stat().st_size new_map.update({name: shard for name in names}) - total += (args.out / shard).stat().st_size print(f"{shard}: {len(names)} tensors", flush=True) for path in sorted(args.src.rglob("*")): diff --git a/scripts/runpod/v41_reap/x1b_pod.sh b/scripts/runpod/v41_reap/x1b_pod.sh new file mode 100755 index 0000000..7686874 --- /dev/null +++ b/scripts/runpod/v41_reap/x1b_pod.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# X1 follow-up on the same pod, after x1_pod.sh: is 96 better picked in one pass on the full model's +# routing (fm96), or in two with the routing re-measured on the 192 it comes from (pf96)? +# Needs /workspace/x1/fm-96.json and the checkpoint already copied to $HF by x1_pod.sh. +set -euo pipefail +W=/workspace; KIT=$W/keep/scripts/runpod/v41_reap; X=$W/x1; HF=${HF:-/root/v41-hf} +export PYTHONPATH=$W/keep/src HF +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$X/x1.log"; } +trap 'status=$?; [[ $status -ne 0 ]] && { log "x1b FAILED (status $status)"; touch "$X/FAILED-B"; }' EXIT +log "x1b: calibration masked to fm-192" +python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/pack/v41-pack.jsonl" --out "$X/mfm192" --sweep calibration \ + --mask "$X/fm-192.json" --shard "${CAL_SHARD:-0/12}" --chunk-tokens 262144 > "$X/cal-mfm192.log" 2>&1 +python3 "$KIT/rerank.py" --stats "$X/mfm192/calibration/sessions" --within "$X/fm-192.json" --keep 96 --out "$X/pf-96.json" | tee -a "$X/x1.log" +log "x1b: eval teacher + fm96, pf96" +python3 "$KIT/collect.py" --hf-path "$HF" --pack "$W/pack/v41-pack.jsonl" --out "$X/eval2" --sweep eval \ + --shard "${EVAL_SHARD:-0/2}" --chunk-tokens 262144 \ + --survivors fm96="$X/fm-96.json" --survivors pf96="$X/pf-96.json" > "$X/eval2.log" 2>&1 +python3 "$KIT/score.py" --teacher "$X/eval2" --student fm96="$X/eval2/student-fm96" --student pf96="$X/eval2/student-pf96" > "$X/scores-b.json" +log "x1b scores written" +touch "$X/DONE-B" diff --git a/tests/test_runpod_v41_reap.py b/tests/test_runpod_v41_reap.py index 875f902..c1b52cb 100644 --- a/tests/test_runpod_v41_reap.py +++ b/tests/test_runpod_v41_reap.py @@ -286,6 +286,19 @@ def test_prune_copies_bytes_renumbers_and_slices(tmp_path): sums = (out / "SHA256SUMS").read_text() assert "tokenizer.json" in sums and "keep-prune.json" in sums + # --skip-shard: an untouched shard (the Engram tables) is indexed but not written + table = "model-00003-of-00003.safetensors" + _write_safetensors(src / table, {"layers.1.engram.embed.weight": ("F8_E4M3", [4, 4], rng.bytes(16))}) + weight_map["layers.1.engram.embed.weight"] = table + (src / prune.INDEX).write_text(json.dumps({"metadata": {"total_size": 0}, "weight_map": weight_map})) + out2 = tmp_path / "out2" + assert prune.main(["--src", str(src), "--survivors", str(survivors), "--out", str(out2), "--skip-shard", table]) == 0 + assert not (out2 / table).exists() and table not in (out2 / "SHA256SUMS").read_text() + assert json.loads((out2 / prune.INDEX).read_text())["weight_map"]["layers.1.engram.embed.weight"] == table + with pytest.raises(ValueError, match="cannot be skipped"): + prune.main(["--src", str(src), "--survivors", str(survivors), "--out", str(tmp_path / "out3"), + "--skip-shard", "model-00001-of-00002.safetensors"]) + # --------------------------------------------------------------------------- pack builder From ab9bd7c58174017d14b0391e02aeeb0545511566 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:38:05 -0800 Subject: [PATCH 40/74] plan: cut the 1-bit base; cancel passes 2b and 3 --- .../2026-11-09-keep-findings-and-levers.md | 6 ++++-- .../2026-11-09-routed-precision-plan.md | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md index fb42fbe..094120d 100644 --- a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md +++ b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md @@ -295,6 +295,8 @@ Predictions registered before the runs that test them (2026-11-09, about 20:40 P 5. On the 2 + 1 ladder, 144 experts at 2 bits plus a routed cache of 96 third-bit residuals (384 units) beats 192 experts at flat 2 bits by at least 0.03 KL. -If 1 holds and 5 fails, the paper is about routing a precision cache and about selection, and the -title changes. If 5 holds, the title stands with the qualifier that the base must be a usable +Predictions 1 and the old-ladder half of the title question will not be run: Jack cut the 1-bit +base the same evening, and passes 2b and 3 were cancelled. Predictions 2 and 3 are in pass 2, +which finishes. If 5 fails, the paper is about routing a precision cache and about selection, and +the title changes. If 5 holds, the title stands with the qualifier that the base must be a usable model. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 502369e..67fd84c 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -17,8 +17,10 @@ Evidence and reasoning for everything below: work runs on the new 192 (`fm-192`). 2. **The ladder moves up one bit.** The 1-bit base is not a usable model (KL 1.358 on its own), so a cache miss is expensive, and nesting on top of it costs 0.053 KL. The main ladder becomes a - 2-bit base with a 1-bit residual (a third bit). The 1-bit + 1-bit ladder stays as the - low-memory regime, measured by the sweeps already queued, with no new work. + 2-bit base with a 1-bit residual (a third bit). The 1-bit base is cut altogether (Jack, + 2026-11-09): at equal memory 96 experts at 2 bits (1.039) beat 192 at 1 bit (1.358), so it + loses at small budgets too. The 1-bit code survives only as the residual. The pass already + running (pass 2) finishes; passes 2b and 3 on the old ladder are cancelled. 3. **Routing the cache is confirmed; "residual beats copy" is not.** A routed cache beats a fixed hot set by 0.068 KL at equal memory and 0.28 GB/s of reads. SSD traffic was not the binding constraint at that point, which removes the residual's main advantage over a separate copy @@ -43,7 +45,7 @@ Claims the paper could stand on, with tonight's evidence against each: | Claim | Status | |---|---| -| 1. The offloaded unit is the nested residual, not a second copy | Open and at risk. Needs R to beat D; with a 1-bit base the nesting penalty (0.053) exceeds the gap to plain pruning (0.025). Retested on the 2 + 1 ladder, where the base is usable. | +| 1. The offloaded unit is the nested residual, not a second copy | Open and at risk. With a 1-bit base the nesting penalty (0.053) exceeds the gap to plain pruning (0.025), and that ladder is cut. Tested on the 2 + 1 ladder, where the base is usable. | | 2. Tier 0 in the same policy | Supported in part: dropping the 7.7% of slots with gate below 0.1 costs 0.001 KL. No joint policy yet. | | 3. The regime at 2 bits and below, 384-expert top-6 model, 128 GB laptop | Narrowed: below 2 bits the base is unusable. The regime is 2 to 3 bits. | | 4. (new) Select experts by total routed mass on routing that resembles the pruned model | Supported: −0.184 KL at 384 → 192 (pod), −0.177 at 192 → 96 (Mac). Not yet searched in the literature. | @@ -69,8 +71,8 @@ third-bit residual 1 unit, a native FP4 expert 4 units. Budgets are per layer. 7. Eval set, version 2: 4,096 tokens per record, and the four sessions that share a first 1,024 tokens no longer counted as four texts. Sub-1-bit comparisons keep the old set for continuity. -Only after step 3 passes is `~/Models/v41-reap50` removable (341 GB); the 1 + 1 sweeps must finish -first. +Only after step 3 passes is `~/Models/v41-reap50` removable (341 GB), and pass 2 must finish +first. `quant/e8-1bit-podimp` and `quant/e8-nest-r1` (66 GB) go with it. ### W0. Related work, read in full (gates the claims; no rubric points) @@ -110,8 +112,10 @@ Runs, on `fm-192`, eval set version 2: 3 bits, all at 2 bits). 2. Pass B, B = 288 and 336: F(144), F(168); R and S at N = 128, K = 32 and 80; N = 112, K = 64. 3. Pass C: D at the K and τ of the best R in each budget; tier 0 on top of the best R. -4. Carried over from revision 1 and already running on REAP-50 (1 + 1 ladder): pass 2 (oracle, - N < 192, projection rungs), pass 2b (separate copies), pass 3 (budgets 240 and 336). +4. Carried over from revision 1 on REAP-50 (1-bit base): pass 2 only (oracle, N < 192, + projection rungs), because it was already an hour in and its oracle result bounds the cache + policy on any ladder. Passes 2b (separate copies) and 3 (budgets 240 and 336) are cancelled; + the separate-copy comparison is made on the new ladder in pass C. Acceptance and scoring inside the item (unchanged in form): @@ -175,7 +179,7 @@ Unchanged, plus a selection section (claim 4) that can also stand alone as a sho | When (estimate) | Work | Score after | |---|---|---:| -| Tonight | 1 + 1 passes 2, 2b, 3 on REAP-50; X1 follow-up; export and download of `fm-192` | 31 | +| Tonight | pass 2 on REAP-50 (last run on the 1-bit base); X1 follow-up; export and download of `fm-192` | 31 | | Day 1 | WR steps 3–7 (capture, fits); W0 reading; W6 bench built on tonight's traces | 31 | | Day 2 | W4 passes A–C on `fm-192`; title decision | 41–46 | | Day 3 | W5 policy and figure; W6 numbers; W7a | 61–66 | From 691f97aac5ba42352959264bd1ed544fd51e6fca Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:42:04 -0800 Subject: [PATCH 41/74] findings: where the selection gain sits (0.052 on teacher-confident positions); plan: generation check before fitting --- .../2026-11-09-keep-findings-and-levers.md | 30 +++++++++++++++++++ .../2026-11-09-routed-precision-plan.md | 9 ++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md index 094120d..57b4b58 100644 --- a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md +++ b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md @@ -216,6 +216,36 @@ Domain mean over 11 domains; pooled over positions in brackets. leaves is different. Working rule: re-measure when the cut removes a large share of mass. The 96-expert follow-up (5.5) tests it. +Where the gain sits (checked the same evening, from the per-position files; `fm192` − `reap192`): + +| Positions | Share | Domain-mean KL difference | 95%, sessions | +|---|---:|---:|---| +| All | 100% | −0.184 | −0.279 to −0.131 | +| Teacher NLL below 0.5 | 64% | −0.026 | −0.047 to +0.002 | +| Teacher NLL below 2 | 74% | −0.052 | −0.080 to −0.023 | +| Teacher NLL 2 or more | 26% | −0.465 | −0.637 to −0.361 | + +- 59% of the pooled gain comes from the 15% of positions where the teacher's NLL is 5 or more, and + 15% from the 74% where it is below 2. On teacher-confident positions top-1 agreement does not + move (0.947 against 0.944 pooled). On tool-use the gain on confident positions is −0.004: all of + it is on positions the teacher did not expect. +- This is not the X2 artifact. On the surprised positions `fm192` agrees with the teacher's top + token more often (0.42 → 0.51) and its NLL moves toward the teacher's (−2.22 → −0.73 relative to + it): it reproduces the full model's expectations better, including its tool-call markup. It is + fidelity on off-format and agentic text, not a general quality gain. +- English-chat is not established as worse: +0.124 over 4,319 positions in 20 short sessions, + interval −0.050 to +0.206, 8 of 20 sessions better. +- "36 of 54 sessions better" is an artifact of the session mix: 30 sessions are english-chat or + english-prose. Every tool-use (7), agentic (4), code-diffs (3), bash (2) and cyber (2) session + improves. +- The four agentic sessions share their first 3,893–3,917 tokens of 16,383 (a common system + prompt), so agentic-python and agentic-rust overlap by a quarter. With the two merged into one + domain the difference is −0.159 (−0.251 to −0.122). +- The claim that survives: at 192 experts, total-mass selection is much closer to the full model + on agentic and tool-call text and modestly closer on ordinary text (−0.05). Whether that is + better for a user needs a generation check (tool-call validity, code completion), which no + number here provides. + ### 5.2 Routed precision, pass 1 (Mac, inside REAP-50, 1-bit base + 1-bit residual) Numbers and intervals are in the recovery log. The decomposition at 288 units per layer: diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 67fd84c..280a7e4 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -48,7 +48,7 @@ Claims the paper could stand on, with tonight's evidence against each: | 1. The offloaded unit is the nested residual, not a second copy | Open and at risk. With a 1-bit base the nesting penalty (0.053) exceeds the gap to plain pruning (0.025), and that ladder is cut. Tested on the 2 + 1 ladder, where the base is usable. | | 2. Tier 0 in the same policy | Supported in part: dropping the 7.7% of slots with gate below 0.1 costs 0.001 KL. No joint policy yet. | | 3. The regime at 2 bits and below, 384-expert top-6 model, 128 GB laptop | Narrowed: below 2 bits the base is unusable. The regime is 2 to 3 bits. | -| 4. (new) Select experts by total routed mass on routing that resembles the pruned model | Supported: −0.184 KL at 384 → 192 (pod), −0.177 at 192 → 96 (Mac). Not yet searched in the literature. | +| 4. (new) Select experts by total routed mass on routing that resembles the pruned model | Supported with a qualifier: −0.184 KL at 384 → 192 (pod), but −0.052 on positions where the teacher is confident; the rest is fidelity on agentic and tool-call text (findings note, 5.1). −0.177 at 192 → 96 (Mac). No generation check yet. Not yet searched in the literature. | ## 3. Workstreams @@ -62,7 +62,12 @@ third-bit residual 1 unit, a native FP4 expert 4 units. Budgets are per layer. shards from `~/Models/v41-reap50` (`cp -c`, no extra space) and check every file against the export's `SHA256SUMS` and REAP-50's. 3. `local_eval.py` against the new checkpoint in BF16/FP4 on the Mac eval set. **Accept** when it - beats REAP-50 on the same sessions by at least 0.10 KL (the pod saw 0.184 on 54 full sessions). + beats REAP-50 on the same sessions by at least 0.10 KL (the pod saw 0.184 on 54 full sessions), + and report the difference on teacher-confident positions separately. +3b. A generation check before any codes are fitted to the new set, since everything so far is + teacher-forced KL: design and size it first (tool-call format validity and code completion on + held-out prompts, new set against REAP-50). Open question: V4.1 has no local decoder on this + Mac, so this may have to run on a pod with the reference implementation. 4. Capture routing and importance on the new model (the step-2c capture, four chunks). 5. Codes: flat 2-bit E8P for all 192 (reuse the fits of the experts shared with REAP-50, 76.5%, if the importance source is the same; otherwise refit all, about 7 h of GPU), then the 1-bit From e8d8530b9cca23ff698bb838369298da2ea5424c Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:56:22 -0800 Subject: [PATCH 42/74] plan revision 3: pruning as a frozen cache, critique, conjectures C1-C10, target architecture, headline proposal; prior art moved to the findings note --- .../2026-11-08-v41-reap-recovery-log.md | 18 + .../2026-11-09-keep-findings-and-levers.md | 43 ++ .../2026-11-09-routed-precision-plan.md | 421 ++++++++---------- 3 files changed, 235 insertions(+), 247 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 8efc10f..cef4333 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -764,3 +764,21 @@ What it says: ladder 2-bit base + 1-bit residual, W4b redefined, sidecars and prune-twice closed, X3 demoted, X8 (domain-balanced selection) added, operating rules for pods and evals. Five predictions are registered in the findings note, section 6, before the runs that test them. + +### 2026-11-09, late: full volume, cancelled sweeps, plan revision 3 + +- Both pod jobs (the 96-expert follow-up and the export) died silently between 04:35 and 04:43 + UTC. Cause: the network volume filled. I sized it in GB against `du` figures in GiB (476 GiB of + model is 511 GB; the 152 GiB export is 163 GB), so 680 GB was 6 GB short. A full volume here + writes zero-length files and returns success, so neither job logged an error, and my status + check (`pgrep -f prune.py` inside an ssh command that itself contains that string) kept reporting + the export as running. About 15 minutes of idle H100. +- Recovery: partial export deleted, volume grown to 720 GB ($50 a month), follow-up eval resumed + alone at 04:51 UTC from its 8 finished sessions (files written before the volume filled were + checked: complete). The export moves to a CPU pod. +- Jack cut the 1-bit base altogether. Passes 2b and 3 cancelled; pass 2 finishes. +- Where the selection gain sits, and the checks behind it: findings note, section 5.1. +- Plan [revision 3](2026-11-09-routed-precision-plan.md): thesis "pruning is a frozen cache", + target architecture, a critique of the approach, ten falsifiable conjectures, eleven ordered + experiments, and a proposed working title conditional on C1. The first experiment (E1) needs no + new fits and runs after pass 2. diff --git a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md index 57b4b58..b45df53 100644 --- a/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md +++ b/docs/deepseek-v4-flash/2026-11-09-keep-findings-and-levers.md @@ -330,3 +330,46 @@ base the same evening, and passes 2b and 3 were cancelled. Predictions 2 and 3 a which finishes. If 5 fails, the paper is about routing a precision cache and about selection, and the title changes. If 5 holds, the title stands with the qualifier that the base must be a usable model. + +### 6.1 Late revision: the lesson I drew from the 1-bit result was too narrow + +Two hours after demoting X3 ("keep every expert, page what is missing") I reversed part of that. +The data refuted a 1-bit copy as the answer to a miss. It did not refute keeping a wide pool of +experts on the SSD. Three measurements point the other way: + +- Pruning is the largest loss in the stack: REAP-50 in BF16 is 0.578 KL from the full model, and + 2-bit coding adds 0.169. The residual work addressed the 0.169. +- A cache that follows the session covers far more demand than a fixed set of the same size: 84–90% + of slots against 75% at 96 of 192, and a routed 48 beat a fixed 96. +- Skipping a low-gate slot costs nothing (+0.001 KL at 7.7% of slots), and static pruning already + answers every miss by re-routing to the best survivor. + +Put together: keep residents at 2 bits, let the resident set follow the session, re-route a miss +among residents, and admit what the router asked for. Static pruning is that system with admission +off. This is conjecture C1 in the plan (revision 3) and it is untested. It was testable tonight +with the existing 2-bit codes and I did not see it, because every design I considered kept a +low-precision copy as the fallback. + +Registered before the run (E1, pool = REAP-50's 192, flat 2 bits, same miss rule for both): + +6. LRU cache of 144 against the static top 144 (0.812): better by 0.03–0.05 KL. +7. LRU cache of 96 against the static top 96 (1.039): better by at least 0.10. +8. Re-route beats drop on the same cache by at least 0.02. +9. The LRU of 144 at 2 bits beats the 1-bit-base residual cache at the same 288 units (0.832). + +## 7. Prior art (moved here from the plan, revision 3) + +Found 2026-11-09. Only abstracts were read for all but MatMoE (introduction and method). + +| Paper | What it does | Overlap with us | +|---|---|---| +| MatMoE (arXiv 2611.01704, ICCAD 2026) | Nested integer codes per expert; bit-width per expert at run time from its routing score | Per-token precision with nested codes. All weights resident; no offload, no pruning. | +| DynaExq (arXiv 2511.15015) | Hot experts resident at high precision, low-precision fallback, asynchronous promotion | A no-stall cache. Separate copies. GPU memory, not SSD. | +| DyMoE (arXiv 2603.19172) | Run-time mixed precision by importance, depth-adaptive, look-ahead prefetch | Depth regimes, prefetch. | +| HOBBIT (arXiv 2411.01433) | On a miss, run a low-precision copy instead of waiting | A low tier as the miss path. | +| MC-MoE (arXiv 2410.06270) | Static mixed precision plus online per-token expert pruning | Per-token dropping with static precision. | +| PMQ, GEMQ (ICML 2026), EdgeMoE | Static per-expert bit allocation | The step 2c allocator. | + +To read in full and to search for (W0): expert caches that re-route or skip on a miss; any +comparison of an expert cache against pruning at equal memory; REAP's comparison of its criterion +against frequency-based ones; calibration on the pruned model's routing. diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 280a7e4..6045372 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -1,248 +1,175 @@ -# Routed precision: implementation plan to 100% (revision 2) - -First written 2026-11-09; revised the same evening after the first routed sweep, the X1 pod run -and the sidecar result. Revision 1 is in git history (`fd3d57e`). Target headline (Jack, -2026-11-09): *Offload the Residual, Not the Expert: Unifying Pruning, Quantization and Offloading in -MoE Inference*. Progress is scored with the fixed rubric in the -[recovery log](2026-11-08-v41-reap-recovery-log.md) ("Headline progress rubric"); the rubric and its -weights do not change with this revision. Score today: **31 / 100**. - -Evidence and reasoning for everything below: -[2026-11-09-keep-findings-and-levers.md](2026-11-09-keep-findings-and-levers.md), sections 5 and 6. - -## 1. What changed, in four lines - -1. **The base model moves.** Ranking experts by total routed mass beats the shipped REAP-50 by - 0.184 KL at 192 experts on the pod's eval. Every Mac number so far sits on REAP-50. All further - work runs on the new 192 (`fm-192`). -2. **The ladder moves up one bit.** The 1-bit base is not a usable model (KL 1.358 on its own), so a - cache miss is expensive, and nesting on top of it costs 0.053 KL. The main ladder becomes a - 2-bit base with a 1-bit residual (a third bit). The 1-bit base is cut altogether (Jack, - 2026-11-09): at equal memory 96 experts at 2 bits (1.039) beat 192 at 1 bit (1.358), so it - loses at small budgets too. The 1-bit code survives only as the residual. The pass already - running (pass 2) finishes; passes 2b and 3 on the old ladder are cancelled. -3. **Routing the cache is confirmed; "residual beats copy" is not.** A routed cache beats a fixed - hot set by 0.068 KL at equal memory and 0.28 GB/s of reads. SSD traffic was not the binding - constraint at that point, which removes the residual's main advantage over a separate copy - there. The separate-copy sweep (tonight) and the 2 + 1 ladder decide the title. -4. **Three levers are closed:** sidecars (+0.199 KL end to end), pruning twice (no gain over one - pass at 192, none at 96), and the tool-use NLL effect (an artifact). - -## 2. Prior art (unchanged from revision 1) - -| Paper | What it does | Overlap with us | +# Routed residency: plan to the paper (revision 3) + +Revised late on 2026-11-09 after a critique of revision 2 (in git at `691f97a`). Score on the fixed +rubric in the [recovery log](2026-11-08-v41-reap-recovery-log.md): **31 / 100**; the rubric and its +weights do not change. Evidence for every number here is in the +[findings note](2026-11-09-keep-findings-and-levers.md); prior art is its section 7. + +## 1. Thesis + +Pruning is a cache that never updates. Keep every resident expert at a usable precision (2 bits), +let the set of resident experts follow the session, and send a token whose expert is absent to the +best resident one instead of to a worse copy. Precision above 2 bits is a second, smaller question. + +This is a conjecture (C1 below), not a result. What is measured so far: a routed cache of +residuals beats a fixed one by 0.068 KL; a 1-bit fallback is the dominant cost; dropping low-gate +slots is free; the second bit is worth more than breadth. + +## 2. Target architecture + +```mermaid +flowchart LR + subgraph OFF[Offline, once] + CAL[Calibration: routing statistics] --> SEL[Rank experts by total routed mass] + SEL --> FIT[2-bit lattice code per expert] + FIT --> PACK[One file extent per expert] + end + subgraph SSD[SSD] + POOL[Expert pool at 2 bits: 192 or 384 per layer] + HIGH[High tier: third bit or native FP4] + ENG[Engram tables, 190 GB] + end + subgraph RAM[Unified memory] + CORE[Always resident: attention, shared experts, router rows for the whole pool] + RES[Resident experts: N per layer at 2 bits] + HOT[Hot tier: H slots, optional] + end + subgraph TOK[Per token, per layer] + R1[Router scores the whole pool] --> R2[Top 6 among resident experts] + R2 --> R3[Skip slots below the gate floor] + R3 --> R4[Run experts and sum] + R1 --> DEM[Demand: wanted but not resident] + end + PACK --> POOL + CORE --> R1 + RES --> R2 + DEM -->|gate at least tau, asynchronous| Q[Admission queue] + POOL --> Q + Q -->|pread, usable from the next token| RES + HIGH -.-> HOT +``` + +- Decode never waits on the SSD: a miss is re-routed now and admitted for later tokens. +- Static pruning is this system with admissions switched off. Per-token dropping is the gate floor. + Quantization is the precision of the resident copy. One budget, in resident GB, covers all three. +- Open in this picture: the pool size (192 or all 384), whether a hot tier pays, and the real + resident size of everything that is not a routed expert. + +## 3. Rules + +1. **Two numbers per comparison.** KL on teacher-confident positions (teacher NLL below 2) is + general quality; agreement on the rest is fidelity on agentic and tool-call text. A change that + moves only the second is reported as that. +2. **Spend memory where it pays.** The second bit is worth about three times more per unit than + the last 48 experts, and a 1-bit base loses at every budget. One sweep asks where the next unit + goes: a resident expert, a third bit, or a native copy. +3. **Use the slack.** The routed cache read 0.28–0.83 GB/s against a 2.5 GB/s ceiling, so whole + 2-bit experts (2 units) and native copies (4 units) are affordable to move. +4. **Fix the fallback before the policy.** No work on admission rules unless the oracle gap exceeds + 0.015 KL. +5. **Fail fast.** Score a partial result end to end before finishing it; write the prediction + before the run; estimate from measured throughput. + +## 4. Critique of the approach so far + +| Hole | Why it matters | What closes it | |---|---|---| -| MatMoE (arXiv 2611.01704, ICCAD 2026) | Nested integer codes per expert; bit-width per expert at run time from its routing score | Per-token precision with nested codes. All weights resident; no offload, no pruning. | -| DynaExq (arXiv 2511.15015) | Hot experts resident at high precision, low-precision fallback, asynchronous promotion | Our no-stall cache. Separate copies. GPU memory, not SSD. | -| DyMoE (arXiv 2603.19172) | Run-time mixed precision by importance, depth-adaptive, look-ahead prefetch | Depth regimes, prefetch. | -| HOBBIT (arXiv 2411.01433) | On a miss, run a low-precision copy instead of waiting | Low tier as the miss path. Separate copies. | -| MC-MoE (arXiv 2410.06270) | Static mixed precision plus online per-token expert pruning | Per-token tier 0 with static precision. | -| PMQ, GEMQ (ICML 2026), EdgeMoE | Static per-expert bit allocation | Our step 2c allocator. | - -Only abstracts were read for all but MatMoE. W0 reads them in full. - -Claims the paper could stand on, with tonight's evidence against each: - -| Claim | Status | -|---|---| -| 1. The offloaded unit is the nested residual, not a second copy | Open and at risk. With a 1-bit base the nesting penalty (0.053) exceeds the gap to plain pruning (0.025), and that ladder is cut. Tested on the 2 + 1 ladder, where the base is usable. | -| 2. Tier 0 in the same policy | Supported in part: dropping the 7.7% of slots with gate below 0.1 costs 0.001 KL. No joint policy yet. | -| 3. The regime at 2 bits and below, 384-expert top-6 model, 128 GB laptop | Narrowed: below 2 bits the base is unusable. The regime is 2 to 3 bits. | -| 4. (new) Select experts by total routed mass on routing that resembles the pruned model | Supported with a qualifier: −0.184 KL at 384 → 192 (pod), but −0.052 on positions where the teacher is confident; the rest is fidelity on agentic and tool-call text (findings note, 5.1). −0.177 at 192 → 96 (Mac). No generation check yet. Not yet searched in the literature. | - -## 3. Workstreams - -Units: one expert at 1 bit per weight = 1 unit = 4.6 MB per layer. A 2-bit expert is 2 units, a -third-bit residual 1 unit, a native FP4 expert 4 units. Budgets are per layer. - -### WR. Re-base on the total-mass 192 (new; gates W4 onward; no rubric points) - -1. Export `fm-192` on the pod (`prune.py --skip-shard` for the two Engram shards, 152 GB). Running. -2. Download through a CPU pod on the volume, 16 parallel streams. On the Mac, clone the two Engram - shards from `~/Models/v41-reap50` (`cp -c`, no extra space) and check every file against the - export's `SHA256SUMS` and REAP-50's. -3. `local_eval.py` against the new checkpoint in BF16/FP4 on the Mac eval set. **Accept** when it - beats REAP-50 on the same sessions by at least 0.10 KL (the pod saw 0.184 on 54 full sessions), - and report the difference on teacher-confident positions separately. -3b. A generation check before any codes are fitted to the new set, since everything so far is - teacher-forced KL: design and size it first (tool-call format validity and code completion on - held-out prompts, new set against REAP-50). Open question: V4.1 has no local decoder on this - Mac, so this may have to run on a pod with the reference implementation. -4. Capture routing and importance on the new model (the step-2c capture, four chunks). -5. Codes: flat 2-bit E8P for all 192 (reuse the fits of the experts shared with REAP-50, 76.5%, if - the importance source is the same; otherwise refit all, about 7 h of GPU), then the 1-bit - residual of the 2-bit base (`--residual-of`, about 4.5 h). -6. Nested expert sets inside the new 192 by total mass on its own routing: 176, 160, 144, 128, 112. -7. Eval set, version 2: 4,096 tokens per record, and the four sessions that share a first 1,024 - tokens no longer counted as four texts. Sub-1-bit comparisons keep the old set for continuity. - -Only after step 3 passes is `~/Models/v41-reap50` removable (341 GB), and pass 2 must finish -first. `quant/e8-1bit-podimp` and `quant/e8-nest-r1` (66 GB) go with it. - -### W0. Related work, read in full (gates the claims; no rubric points) - -Unchanged. Adds a search for claim 4 (frequency-weighted or mass-based expert pruning criteria, -and calibration on the pruned model's routing). - -### W1. Nested codes validated (rubric item 1: done, 10) - -Fit complete, 40 of 40 layers; base + residual error 0.102 / 0.084 against 0.090 flat; end to end -+0.053 KL [+0.039, +0.071]. The acceptance bound (all-tier-2 KL at most 0.80) was met at 0.8001. -The alternating refit is no longer planned for the 1 + 1 ladder; it is used on the 2 + 1 ladder -only if its nesting penalty exceeds 0.02 KL. - -### W4. Core result (rubric item 4: 0 → 25) - -The claim: at equal resident memory, a routed cache of third-bit residuals over a 2-bit base gives -lower KL than any static assignment and than a separate-copy cache, at an SSD read rate the Mac -sustains. - -Budgets B = 288, 336 and 384 units (53, 62, 71 GB of routed experts). A configuration keeps N -experts at 2 bits and K residual slots, 2N + K = B. - -| Policy | What it is | -|---|---| -| F, flat | B / 2 experts at 2 bits, nothing else. The frontier to beat. | -| S, static-nested | fixed hot set of K residuals by routed mass | -| R, routed-nested (ours) | LRU of K residuals, asynchronous admission at gate ≥ τ | -| D, separate-copy | the hot expert's high tier is its native FP4 copy (4 units, swapped for the 2-bit copy: 2 extra units resident, 6 units moved per promote/demote cycle against 1) | -| O, per-session oracle | hot set = that record's own top K; the ceiling for any cache | - -Build: a base-unit field in `routed_report.py` (bases are no longer 1 unit); a native-precision -high tier for `--dynamic BASE|HI`; `--tau-file` (W5). - -Runs, on `fm-192`, eval set version 2: - -1. Pass A, B = 384: F(192); R, S, O at N = 160, K = 64 and N = 144, K = 96; endpoints (all at - 3 bits, all at 2 bits). -2. Pass B, B = 288 and 336: F(144), F(168); R and S at N = 128, K = 32 and 80; N = 112, K = 64. -3. Pass C: D at the K and τ of the best R in each budget; tier 0 on top of the best R. -4. Carried over from revision 1 on REAP-50 (1-bit base): pass 2 only (oracle, N < 192, - projection rungs), because it was already an hour in and its oracle result bounds the cache - policy on any ladder. Passes 2b (separate copies) and 3 (budgets 240 and 336) are cancelled; - the separate-copy comparison is made on the new ladder in pass C. - -Acceptance and scoring inside the item (unchanged in form): - -- 10 points: R beats F and S by at least 0.03 KL and beats D at equal memory and equal SSD rate, - session-level bootstrap 95% interval excluding zero, at one budget, with SSD traffic at or - below 2.5 GB/s at 30 tok/s. -- 15 points: the same at two of three budgets on eval set version 2. -- 25 points: reproduced on the pod's eval (108 sessions, full length) with the routed forward on - GPU (W4b). - -W4b is redefined: it no longer fits codes for 384 experts (X3 is demoted, see section 5). It ports -the routed forward to `collect.py`, uploads the Mac's codes, and scores the winning configurations -on the pod. Estimate from tonight's throughput (about 11 minutes per stream over 54 sessions): -2–3 h on one H100, under $15. - -If R does not beat F on the 2 + 1 ladder either: - -1. Admission by prediction (score layer l + 1's gate on layer l's output), measured offline first. -2. Report the routed-cache result against S and D as the contribution, with the selection finding, - under a title that does not claim to beat pruning. Say so then; do not stretch the claim. - -### W5. Unification (rubric item 5: 0 → 10) - -Unchanged in design (one benefit-per-byte rule with one λ; `routed_policy.py`, -`routed_frontier.py`). Changes: tier 0 enters with tonight's measurement as its first point; the -sidecar tier is removed; the frontier figure plots both ladders, with F, S, R, D and O. - -### W6. Real Apple Silicon streaming (rubric item 6: 0 → 10) - -Unchanged in design. Two notes from tonight: the admission rates to replay are low (9.5 to 28 MB -per token at K = 96), so the bar is latency per admission, not sustained throughput; and the -network-volume episode is the same failure in miniature, a single reader faulting pages one at a -time, so the bench measures `pread` from a thread pool against `mmap` faults (X4) on purpose. - -### W7. Learned policy and tier-aware codes (rubric item 7: 0 → 10) - -- (a) 5 points: per-(layer, expert) thresholds from W5 beat one global τ by at least 0.02 KL. -- (b) 5 points: a jointly fitted nested code (base chosen so that base + residual is best, not the - base alone) or tier-aware importance, gaining at least 0.02 KL at equal units. - -### W8. Generality (rubric item 8: 0 → 10) - -Unchanged (Qwen3-30B-A3B, download needs Jack's go). Adds the selection rule as a second thing to -replicate there, since it needs only routing statistics. - -### W9. Paper (rubric item 9: 1 → 5) - -Unchanged, plus a selection section (claim 4) that can also stand alone as a short note. - -### WS. Selection, to its own conclusion (new; supports W9 and a possible REAP-50 v2) - -- Tonight: 96 experts in one pass on full-model routing against two passes through `fm-192` - (`x1b_pod.sh`), to test the rule "re-measure routing when the cut removes a large share of mass". -- X8, domain-balanced mass: calibration tokens are uneven by domain (english-chat 52K, tool-use - 451K) and english-chat is the one domain that got worse (0.448 → 0.572). Needs per-domain - accumulation in `collect.py` and one calibration pass on a pod; run with W4b. Adopt only if it - gains at least 0.03 domain-mean KL, because adopting it means re-basing again. -- Publishing a replacement REAP-50 on Hugging Face is Jack's decision. - -## 4. Order and expected score - -| When (estimate) | Work | Score after | -|---|---|---:| -| Tonight | pass 2 on REAP-50 (last run on the 1-bit base); X1 follow-up; export and download of `fm-192` | 31 | -| Day 1 | WR steps 3–7 (capture, fits); W0 reading; W6 bench built on tonight's traces | 31 | -| Day 2 | W4 passes A–C on `fm-192`; title decision | 41–46 | -| Day 3 | W5 policy and figure; W6 numbers; W7a | 61–66 | -| Day 4 | W4b and X8 on a pod; W7b | 76–81 | -| Days 5–6 | W8 | 91–96 | -| Day 7 | W9 full draft | 100 | - -The score moves only on measured results, so a failed acceptance test holds it where it is. The -re-base costs about a day against revision 1's schedule and raises every number it touches. - -## 5. Levers: status after tonight - -| # | Lever | Status | -|---|---|---| -| X1 | Prune twice at 384 → 192 | Resolved. The ranking rule is the gain (−0.184 KL); the second pass is not (+0.014 at 192, −0.009 n.s. at 96). The second pass used 12× less calibration data than the one-pass table, so "slightly worse" is not separable from sample size. | -| X2 | Pruning as specialization | Closed (artifact). | -| X3 | Keep all 384 bases at 1 bit, offload residuals | Demoted. A 1-bit base scores 1.358 alone and misses fall to it. At 2 bits, 384 bases are 768 units, over any budget here. | -| X4 | OS page cache as the precision cache | Open, in W6. | -| X5 | Stronger base, finer rungs | (b) promoted: the 2 + 1 ladder is now W4's main line. (a) projection rungs are in pass 2. | -| X6 | Budgets by sensitivity | Open, after W4. | -| X7 | Shared base across experts | Open, low priority. | -| X8 | Domain-balanced selection | New, in WS. | -| — | Sidecars (old step 3) | Closed: +0.199 KL [+0.164, +0.219] with 15 layers fitted. | - -## 6. Operating rules learned tonight - -Pods: -- Never run the model from a network volume: one stream reads 120 MB/s, a random row takes - 2–3 ms. Create the GPU pod with a container disk that holds the checkpoint and copy first - (16 streams, 476 GB in 7 minutes). -- Smoke-test every launcher on the image before the GPU clock matters; match readiness by - substring; `PIP_BREAK_SYSTEM_PACKAGES=1` on Ubuntu 24.04 images. -- Estimates from measured throughput: about 390 tok/s for a calibration stream and 11 minutes per - eval stream over 54 sessions. The X1 estimate (6.5–7.5 h) was 4× too high. -- CPU-only work (exports, transfers) runs on a CPU pod or alongside a GPU job that is already - running, never on an idle GPU. - -Evaluation: -- Report session-level bootstrap intervals; position-level intervals are about half as wide. -- Domain mean and pooled KL differ a lot (0.716 against 0.950 for REAP-50) because the agentic and - tool-use domains are long and hard. State which one a number is. -- Mac numbers (22 sessions, first 1,024 tokens) and pod numbers (54 sessions, full length) are not - comparable in absolute terms; only differences within one set are. -- A layer-local metric is not evidence. Twice now (GLM-4.5-Air, V4.1) a sidecar that improved its - layer objective failed end to end. Score a partial fit end to end before finishing it. -- Settle the expert set before fitting codes to it. -- Write the prediction down before the run (findings note, section 6). - -## 7. Resources - -- Memory: 80 GB cap; the 8-variant sweep peaked at 64.1 GB. -- Disk: 373 GB freed on 2026-11-09 (REAP-75, the no-effect refit, sidecar files). The new - checkpoint adds 152 GB; its codes about 100 GB. -- Pod: about $20 spent tonight (H100 about 4 h, CPU pod, volume). Volume `yqazs7aqe6` is 680 GB - ($48 a month) and holds the full model and the export; delete it after W4b. - -## 8. Decisions for Jack - -1. Title: decide after the separate-copy sweep and the first 2 + 1 pass. Until then it stands. -2. A replacement REAP-50 on Hugging Face built from the total-mass table (public change). -3. Qwen3-30B-A3B download for W8. +| We never tested the simplest design: a 2-bit expert cache with re-routing on a miss. We were anchored on residuals. | Pruning costs 0.578 KL and 2-bit coding 0.169; the residual work chased the smaller loss with a fallback that could not work. | E1, tonight, no new fits. | +| Every result is teacher-forced KL. No text has been generated. | A 0.03 KL threshold is not tied to anything a user sees; the selection gain sits mostly on positions the teacher did not expect. | Rule 1 now; a generation check (E6) before any public claim. | +| The offload result is a simulation, and V4.1 has no decoder on this Mac. | "On a laptop" rests on a trace replay, an assumed 30 tok/s, and an SSD shared with 190 GB of Engram lookups. | W6 replay states exactly that; real decode only on the second model (E8). A stated limitation. | +| The memory axis counts routed experts only. | Attention, the KV cache and the Engram page cache are resident too, and this Mac thrashed at 91 GB. The true budget may sit below 384 units. | Measure total resident size once; plot the frontier in total GB. | +| τ and K were picked on the eval set. | Selection bias in our own favour. | Tune on the Mac set; confirm once on the pod's untouched 54 sessions (shard 1/2). | +| The eval set is thin: domains with one session, shared prompts, sessions of 11 tokens. | Domain means swing on one text. | Eval set version 2; session-level intervals; all 108 sessions for the final table. | +| The static baseline was REAP-50. | A better static set exists (total mass), so a win over REAP-50 flatters the method. | Static baselines use the total-mass set after the re-base. | +| The third-bit bet has a thin margin. | Everything above 2 bits is worth at most 0.169 KL on this pool. | C3 is tested after C1 and C2, not before. | +| The selection claim may contradict REAP's own comparisons against frequency-based criteria. | A reviewer will know that paper. | W0 reads it; C7 adds a generation test. To verify: this is from memory of the paper. | +| Three avoidable pod failures in one evening, the last a volume filled by mixing GB and GiB. | About $8 and two hours. | Section 9. | + +## 5. Conjectures the paper proves or refutes + +Each is stated so that a single run can refute it. KL is domain-mean; "confident" means positions +with teacher NLL below 2; intervals are session-level bootstrap. + +| # | Conjecture | Test | Refuted if | +|---|---|---|---| +| C1 | **Residency should follow the session.** With the same codes, memory and miss rule, an LRU expert cache with demand admission beats the best static expert set. | E1: pool of 192 at flat 2 bits, N = 144 and 96, static top-N in the same run. | Gain below 0.02 confident KL at both N, or interval includes zero, or reads above 2.5 GB/s at 30 tok/s. | +| C2 | **A wider pool beats a better static set.** At 384 units, 192 residents drawn from all 384 experts beat the best static 192. | E3–E4: 2-bit codes for all 384, cache of 192. | Gain below 0.03 confident KL. | +| C3 | **Above 2 bits, precision should be routed too.** At equal units, a hot tier (third bit or native FP4) on the hottest residents beats spending those units on more 2-bit residents. | E5 on the winner of C1/C2. | Gain below 0.02, or negative. My prior is close to even. | +| C4 | **Re-route beats drop beats a 1-bit copy** as the answer to a miss, at equal memory. | E1: same cache with drop and with re-route; the 1-bit-base cache from pass 1 (0.832 at 288 units). | Any other order. | +| C5 | **The policy is near its ceiling.** A per-session oracle set beats the LRU by at most 0.015 KL. | Pass 2 (residual cache), E1 (expert cache). | Gap above 0.015: then prediction and prefetch are worth building. | +| C6 | **Traffic does not bind.** The best configuration at each budget reads under 1 GB/s at 30 tok/s, and 99% of admissions land within one token interval on the internal SSD. | Simulation in E1/E4; W6 replay. | Either number missed: then the unit moved matters and nested residuals return. | +| C7 | **Rank by sum, not mean.** Total routed mass beats REAP's per-route score (a) on confident KL and (b) in generated tool calls and code. | (a) measured at 192: −0.052 [−0.080, −0.023]. (b) E6. | (b) shows no gain or a loss. | +| C8 | **Re-measure routing when the cut is deep.** 96 experts picked through the 192 beat 96 picked in one pass by at least 0.05. | E2, on the pod now. | Smaller, or reversed. | +| C9 | **The gate floor is free.** Skipping slots with gate below 0.1 costs under 0.005 KL in any configuration. | One point so far (+0.001); added to E1 and E4. | Any configuration above 0.005. | +| C10 | **C1 holds on a second MoE.** | E8, Qwen3-30B-A3B, with real generation. | C1's test fails there. | + +C1, C2 and C6 carry the title. C7–C8 are a second, independent result. + +## 6. Experiments, in order + +| # | What | Where, cost | Gate | +|---|---|---|---| +| E0 | Pass 2 on the old ladder (oracle gap for C5). | Mac, finishing about 23:00. | — | +| E1 | Expert cache at 2 bits inside REAP-50: LRU and oracle at N = 144 and 96, static controls, drop against re-route, gate floor. Needs `--miss drop\|reroute` in `local_eval.py` and stratified columns in `routed_report.py`. | Mac, about 3.5 h, starts after E0. | Decides C1, C4, C5. | +| E2 | 96 experts, one pass against two. | Pod, running. | C8. | +| WR | Export and download the total-mass 192: the honest static baseline and a better pool of 192. | CPU pod, overnight. | — | +| E3 | If C1 holds: 2-bit codes for all 384 experts and the full router rows, as a codes-only pool (about 150 GB). Size the fit first; it has only run on the Mac GPU. | Pod. | Needs C1. | +| E4 | Cache of 192 and 144 drawn from 384. | Mac. | C2, C6. | +| E5 | Hot tier on the winner: third bit (needs one residual fit) and native FP4 (no fit). | Mac. | C3. | +| E6 | Generation check: tool-call validity and code completion, total-mass set against REAP-50, then the best cache. Not designed yet; needs the reference decoder on a pod. | Pod. | C7b; gates any public release. | +| W6 | Trace replay on the internal SSD, `pread` pool against `mmap` faults, with and without memory pressure. | Mac. | C6. | +| E7 | One confirmation on the pod's untouched 54 sessions with the routed forward on GPU. | Pod, 2–3 h. | Item 4, 25 points. | +| E8 | Second model. | Mac; download needs Jack's go. | C10. | + +If C1 fails: the static path stands (re-base, C3 on a static set), the title falls back, and the +paper is the unit accounting plus the selection result. + +## 7. Headline + +The current title, *Offload the Residual, Not the Expert*, is not defensible on tonight's data: the +1-bit base it needs is cut, SSD traffic did not bind where it was measured, and the design in +section 2 offloads experts. It survives only if C6 is refuted. + +Proposed working title, conditional on C1: + +> **Pruning Is a Frozen Cache: Unifying Pruning, Quantization and Offloading in MoE Inference** + +It keeps the half of the original headline that has held up, and it states the claim a reader can +check: a static expert set is the special case of a routed one, and it is dominated at equal +memory. Fallback if C1 fails: *Bits Before Breadth: What a Gigabyte Buys in MoE Compression*, an +empirical paper on the unit accounting and the selection rule, which the data already supports. + +Novelty risk: expert caches with a miss fallback exist (HOBBIT, DynaExq, and gating that skips +experts). What they do not appear to do is compare against pruning at equal memory, or treat the +two as one policy. W0 must confirm that before the title is used. + +## 8. Rubric mapping + +| Item | Now | Moves when | +|---|---:|---| +| 1 Nested codes validated | 10 | done (and the 1-bit ladder is retired) | +| 2 Per-token tiers built | 10 | done | +| 3 Cache simulation and cost model | 10 | done | +| 4 Core result (25) | 0 | 10: the routed cache beats the best static set by 0.03 KL (0.02 confident) and the 1-bit-fallback cache, one budget, reads under 2.5 GB/s. 15: two budgets, eval set 2, total-mass baseline. 25: E7. | +| 5 Unification and frontier figure (10) | 0 | one policy sets residency, gate floor and tier; figure in total resident GB | +| 6 Real Apple Silicon streaming (10) | 0 | W6 | +| 7 Learned policy or trained codes (10) | 0 | per-expert thresholds, or a hot tier that passes C3 | +| 8 Second MoE and downstream (10) | 0 | E8, E6 | +| 9 Paper draft (5) | 1 | sections as conjectures resolve | + +## 9. Operating rules and resources + +- Pods: copy the checkpoint to local disk before GPU work; size volumes in GiB with 10% headroom + and check `df` before a large write (a full network volume writes empty files without an error); + never run a second large job beside a GPU job; test launchers before the GPU clock matters. +- Evals: session-level intervals; state domain mean or pooled; no layer-local metric as evidence; + settle the expert pool before fitting codes to it. +- Spend tonight: about $20 (H100 4.5 h, CPU pod, volume). Volume `yqazs7aqe6` is 720 GB, $50 a + month, deleted after E7. +- Mac: 80 GB cap; 469 GB free and rising after the cleanup. + +## 10. Decisions for Jack + +1. The title in section 7, after E1. +2. A replacement REAP-50 on Hugging Face: not before E6. +3. Qwen3-30B-A3B download for E8. From 0cdfacc0107dfbcba2a579df7ec4b5ee0f405653 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:59:12 -0800 Subject: [PATCH 43/74] local_eval: expert cache with drop or re-route on a miss (--miss), gate_masked, resident-set output; tests; E1 queued --- .../2026-11-09-routed-precision-plan.md | 4 +- scripts/runpod/v41_reap/local_eval.py | 58 +++++++++++++++++-- scripts/runpod/v41_reap/routed_report.py | 3 +- tests/test_runpod_v41_routed_precision.py | 36 ++++++++++++ 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 6045372..903ee86 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -100,7 +100,7 @@ with teacher NLL below 2; intervals are session-level bootstrap. | C6 | **Traffic does not bind.** The best configuration at each budget reads under 1 GB/s at 30 tok/s, and 99% of admissions land within one token interval on the internal SSD. | Simulation in E1/E4; W6 replay. | Either number missed: then the unit moved matters and nested residuals return. | | C7 | **Rank by sum, not mean.** Total routed mass beats REAP's per-route score (a) on confident KL and (b) in generated tool calls and code. | (a) measured at 192: −0.052 [−0.080, −0.023]. (b) E6. | (b) shows no gain or a loss. | | C8 | **Re-measure routing when the cut is deep.** 96 experts picked through the 192 beat 96 picked in one pass by at least 0.05. | E2, on the pod now. | Smaller, or reversed. | -| C9 | **The gate floor is free.** Skipping slots with gate below 0.1 costs under 0.005 KL in any configuration. | One point so far (+0.001); added to E1 and E4. | Any configuration above 0.005. | +| C9 | **The gate floor is free.** Skipping slots with gate below 0.1 costs under 0.005 KL in any configuration. | One point so far (+0.001); added to E4. | Any configuration above 0.005. | | C10 | **C1 holds on a second MoE.** | E8, Qwen3-30B-A3B, with real generation. | C1's test fails there. | C1, C2 and C6 carry the title. C7–C8 are a second, independent result. @@ -110,7 +110,7 @@ C1, C2 and C6 carry the title. C7–C8 are a second, independent result. | # | What | Where, cost | Gate | |---|---|---|---| | E0 | Pass 2 on the old ladder (oracle gap for C5). | Mac, finishing about 23:00. | — | -| E1 | Expert cache at 2 bits inside REAP-50: LRU and oracle at N = 144 and 96, static controls, drop against re-route, gate floor. Needs `--miss drop\|reroute` in `local_eval.py` and stratified columns in `routed_report.py`. | Mac, about 3.5 h, starts after E0. | Decides C1, C4, C5. | +| E1 | Expert cache at 2 bits inside REAP-50 (`local_eval.py --miss drop\|reroute`, built and unit-tested). Eight variants: static, LRU (τ 0.2 and 0.1), oracle and drop at N = 144; static, LRU and oracle at N = 96. The static variants use exactly the sets scored earlier, so they must reproduce 0.812 and 1.039: that is the end-to-end check of the new path. Queued (`jobs/e1.sh`) behind E0 with a 64-token smoke test. Stratified columns in `routed_report.py` are still to build. | Mac, about 3.5 h. | Decides C1, C4, C5. | | E2 | 96 experts, one pass against two. | Pod, running. | C8. | | WR | Export and download the total-mass 192: the honest static baseline and a better pool of 192. | CPU pod, overnight. | — | | E3 | If C1 holds: 2-bit codes for all 384 experts and the full router rows, as a codes-only pool (about 150 GB). Size the fit first; it has only run on the Mac GPU. | Pod. | Needs C1. | diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index be4f243..24901e8 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -66,6 +66,30 @@ def local_masks(spec: str, ckpt_survivors: dict[str, list[int]]) -> dict[int, np return masks +def gate_masked(gate, x, allowed, linear): + """``Gate.forward`` with a per-token candidate set: ``allowed`` [n, n_experts] bool. + + The router picks its top-k among the allowed experts and normalizes over those, which is what + a router masked to a pruned set does, here with a set that changes from token to token. A row + with fewer than top-k allowed experts gives its surplus slots zero weight.""" + + import torch + import torch.nn.functional as F + + scores = linear(x.float(), gate.weight.float()) / gate.gate_temp + if gate.score_func == "softmax": + scores = scores.softmax(dim=-1) + elif gate.score_func == "sigmoid": + scores = scores.sigmoid() + else: + scores = F.softplus(scores).sqrt() + indices = (scores + gate.bias).masked_fill(~allowed, float("-inf")).topk(gate.topk, dim=-1)[1] + weights = scores.gather(1, indices) * allowed.gather(1, indices) + if gate.norm_topk_prob and gate.topk > 1: + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) + return weights * gate.route_scale, indices + + def expert_override(spec: str, n_experts: int, ckpt_survivors: dict): """``layer -> {(expert, proj): bf16 [out, in]}`` from FP4 (``fp4-bf16``) or E8P codes.""" @@ -128,7 +152,7 @@ def mixed(layer: int, sd: dict) -> dict: def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: list[int], warm: np.ndarray | None, counts: np.ndarray, policy: str = "lru", - trace: list | None = None) -> np.ndarray: + trace: list | None = None, resident: np.ndarray | None = None) -> np.ndarray: """Which (token, slot) pairs may run base + residual, under an LRU of ``k`` resident residuals. ``ind``/``wts``: [T, top_k] routed experts and gate weights, the records of ``bounds`` back to @@ -139,7 +163,10 @@ def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: them as (record, token in record, expert). ``policy="oracle"`` is the upper bound for any cache of ``k``: each record's hot set is its own - top ``k`` experts by routed gate weight, known in advance and never changed (no admissions).""" + top ``k`` experts by routed gate weight, known in advance and never changed (no admissions). + + ``resident`` ([T, n_experts] bool, zeroed by the caller) receives the resident set each token + sees, for callers that re-route a miss among resident experts.""" from collections import OrderedDict @@ -150,12 +177,16 @@ def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: mass = np.bincount(ind[start:start + n].ravel(), weights=wts[start:start + n].ravel(), minlength=counts.shape[0]) hot = np.argsort(-mass, kind="stable")[:k] hi[start:start + n] = np.isin(ind[start:start + n], hot[mass[hot] > 0]) + if resident is not None: + resident[start:start + n, hot] = True start += n continue lru = OrderedDict((int(e), None) for e in (warm if warm is not None else [])) for t in range(start, start + n): row = ind[t].tolist() hi[t] = [e in lru for e in row] + if resident is not None: + resident[t, list(lru)] = True for s, e in enumerate(row): if wts[t, s] >= tau: if e in lru: @@ -197,6 +228,9 @@ def main(argv: list[str] | None = None) -> int: help="with --dynamic, per variant: a slot runs base + residual when its gate weight >= VALUE") p.add_argument("--tau-drop", action="append", default=[], metavar="NAME=VALUE", help="with --dynamic, per variant: a slot with gate weight < VALUE is skipped (rung 0 per token)") + p.add_argument("--miss", action="append", default=[], metavar="NAME=drop|reroute", + help="with --cache-k and BASE|HI codes: no base copy. A slot whose expert is not resident is " + "skipped (drop), or the router picks its top-k among resident experts (reroute)") p.add_argument("--cache-k", action="append", default=[], metavar="NAME=K", help="with --dynamic: hi tier only from an LRU of K resident residuals per layer, admitted at gate >= tau") p.add_argument("--cache-warm", type=Path, help="imatrix npz (reap__gate_norm_sum): warm-start each cache with the top-K") @@ -259,6 +293,7 @@ def main(argv: list[str] | None = None) -> int: from keep.vq.e8 import e8_1bit_grid, e8p_full_grid copy_mode = "|" in args.dynamic # the high tier is its own copy, not base + residual + args_cache_k_names = {s.partition("=")[0] for s in args.cache_k} base_dir, _, res_dir = args.dynamic.partition("|" if copy_mode else "+") base_dir, res_dir = Path(base_dir).expanduser(), Path(res_dir).expanduser() hi_proj = {name: {"w2": ("w2",), "w1w3": ("w1", "w3")}[v] for name, _, v in (s.partition("=") for s in args.hi_proj)} @@ -267,6 +302,9 @@ def main(argv: list[str] | None = None) -> int: traces: dict[str, list] = {name: [] for name in args.trace} taus = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau)} tau_drop = {name: float(t) for name, _, t in (s.partition("=") for s in args.tau_drop)} + miss = {name: mode for name, _, mode in (s.partition("=") for s in args.miss)} + assert all(m in ("drop", "reroute") and copy_mode and v in args_cache_k_names for v, m in miss.items()), \ + "--miss takes drop or reroute, and needs BASE|HI codes and --cache-k for the same variant" assert set(taus) == set(variants), "--tau needs one NAME=VALUE per --variant" np_grids = {16: e8p_full_grid().astype(np.float32), 8: e8_1bit_grid().astype(np.float32)} cache: dict = {"layer": None} @@ -278,10 +316,10 @@ def main(argv: list[str] | None = None) -> int: for v in variants: # [layer, expert, (lo slots, hi slots, lo mass, hi mass, admissions, dropped slots, dropped mass)] dyn_counts[v] = np.zeros((n_layers_, n_local, 7)) - def simulate_cache(layer, ind, wts, k, tau, counts): + def simulate_cache(layer, ind, wts, k, tau, counts, resident=None): v, found = state["variant"], [] hi = residual_cache(ind, wts, k, tau, state["bounds"], None if warm is None else warm[layer, :k], counts, - policy=policies.get(v, "lru"), trace=found if v in traces else None) + policy=policies.get(v, "lru"), trace=found if v in traces else None, resident=resident) if v in traces: traces[v].extend((layer, *row) for row in found) return hi @@ -322,13 +360,20 @@ def dynamic_forward(self, x, image_mask=None): y = torch.zeros_like(x, dtype=torch.float32) limit = self.shared_experts.swiglu_limit k = cache_k.get(state["variant"]) + on_miss = miss.get(state["variant"]) + resident = np.zeros((x.size(0), self.gate.weight.size(0)), dtype=bool) if on_miss == "reroute" else None hi_all = None if k is None else torch.as_tensor( - simulate_cache(layer, indices.numpy(), weights.float().numpy(), k, tau, counts)) + simulate_cache(layer, indices.numpy(), weights.float().numpy(), k, tau, counts, resident)) + if resident is not None: # demand (above) drove the cache; the tokens run on what is resident + weights, indices = gate_masked(self.gate, x, torch.as_tensor(resident), ref.linear) + hi_all = weights > 0 for i in torch.unique(indices).tolist(): idx, top = torch.where(indices == i) w = weights[idx, top] hi_mask = w >= tau if hi_all is None else hi_all[idx, top] drop = w < tau_drop.get(state["variant"], 0.0) # rung 0 per token: the slot is skipped + if on_miss: + drop = drop | ~hi_mask # no base copy: a slot that is not resident is not run counts[i, 5] += int(drop.sum()) counts[i, 6] += float(w[drop].sum()) hi_mask = hi_mask & ~drop @@ -487,7 +532,8 @@ def before_moe(layer: int, block, key) -> None: hi_units = 2.0 if copy_mode else len(hi_proj.get(v, ("w1", "w2", "w3"))) / 3 summary["variants"][v]["_tiers"] = {"tau": taus[v], "cache_k": cache_k.get(v), "policy": policies.get(v, "lru") if v in cache_k else None, - "copy_mode": copy_mode, "hi_units": hi_units, "n_bases": kept, + "copy_mode": copy_mode, "hi_units": hi_units, + "n_bases": 0 if v in miss else kept, "pool": kept, "miss": miss.get(v), "tau_drop": tau_drop.get(v, 0.0), "hi_slot_share": float(tot[1] / (tot[:2].sum() + tot[5])), "drop_slot_share": float(tot[5] / (tot[:2].sum() + tot[5])), diff --git a/scripts/runpod/v41_reap/routed_report.py b/scripts/runpod/v41_reap/routed_report.py index 8f58921..ca8dc27 100644 --- a/scripts/runpod/v41_reap/routed_report.py +++ b/scripts/runpod/v41_reap/routed_report.py @@ -45,13 +45,14 @@ def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: hi_read = hi_units if tiers.get("copy_mode") else 1 + hi_units # a separate copy is read alone read = (c[:, :, 0].sum() + c[:, :, 1].sum() * hi_read) * UNIT_BYTES / tokens ssd = c[:, :, 4].sum() * hi_units * UNIT_BYTES / tokens if k is not None else 0.0 - if tiers.get("copy_mode") and k is not None: + if tiers.get("copy_mode") and k is not None and not tiers.get("miss"): # A separate-copy cache can instead drop a hot expert's low copy (1 extra unit, not 2) and read # it back on demotion: every promote/demote cycle then moves 3 units where a residual moves 1. row |= {"units_per_layer_swap": round(n_bases + k, 1), "ssd_gbs_swap": round(c[:, :, 4].sum() * 3 * UNIT_BYTES / tokens * tok_s / 1e9, 2)} row |= { "tau": tiers["tau"], "tau_drop": tiers.get("tau_drop", 0.0), "cache_k": k, "policy": tiers.get("policy"), + "miss": tiers.get("miss"), "n_bases": n_bases, "hi_units": round(hi_units, 2), "hi_slots": round(tiers["hi_slot_share"], 3), "drop_slots": round(tiers.get("drop_slot_share", 0.0), 3), "units_per_layer": round(units, 1), "resident_gb": round(units * n_layers * UNIT_BYTES / 1e9, 1), diff --git a/tests/test_runpod_v41_routed_precision.py b/tests/test_runpod_v41_routed_precision.py index 662f2d9..d14905a 100644 --- a/tests/test_runpod_v41_routed_precision.py +++ b/tests/test_runpod_v41_routed_precision.py @@ -90,3 +90,39 @@ def test_compare_bootstrap(tmp_path): (tmp_path / "summary.json").write_text(json.dumps({"variants": {"a": sessions, "b": sessions}})) out = routed_report.compare(f"{tmp_path}:a", f"{tmp_path}:b", reps=200) assert abs(out["kl_diff"] - 0.1) < 1e-9 and out["ci95"][0] <= 0.1 <= out["ci95"][1] + + +def test_residual_cache_reports_the_resident_set(): + ind = np.array([[0, 1], [2, 1], [2, 0]]) + wts = np.array([[0.9, 0.1], [0.5, 0.1], [0.5, 0.2]]) + resident = np.zeros((3, 4), dtype=bool) + local_eval.residual_cache(ind, wts, k=2, tau=0.3, bounds=[3], warm=np.array([3, 1]), counts=np.zeros((4, 5)), + resident=resident) + # token 0 sees the warm set {3, 1}; expert 0 is admitted and evicts 3; then 2 evicts 1 + assert resident.tolist() == [[False, True, False, True], [True, True, False, False], [True, False, True, False]] + resident[:] = False + local_eval.residual_cache(ind, wts, k=2, tau=0.3, bounds=[3], warm=None, counts=np.zeros((4, 5)), + policy="oracle", resident=resident) + assert resident.all(axis=0).tolist() == [True, False, True, False] # the record's top two by mass: 0 and 2 + + +def test_gate_masked_routes_among_allowed_experts(): + torch = pytest.importorskip("torch") + import types + + torch.manual_seed(0) + gate = types.SimpleNamespace(weight=torch.randn(8, 5), bias=torch.zeros(8), gate_temp=1.0, score_func="sqrtsoftplus", + topk=3, norm_topk_prob=True, route_scale=1.5) + x = torch.randn(4, 5) + scores = torch.nn.functional.softplus(x @ gate.weight.T).sqrt() + w, i = local_eval.gate_masked(gate, x, torch.ones(4, 8, dtype=torch.bool), torch.nn.functional.linear) + assert torch.equal(i, scores.topk(3, dim=-1)[1]) # nothing masked: the reference router + assert torch.allclose(w.sum(-1), torch.full((4,), 1.5)) + allowed = torch.ones(4, 8, dtype=torch.bool) + allowed[:, i[0, 0]] = False + w2, i2 = local_eval.gate_masked(gate, x, allowed, torch.nn.functional.linear) + assert not (i2 == i[0, 0]).any() and torch.allclose(w2.sum(-1), torch.full((4,), 1.5)) # re-routed, renormalized + few = torch.zeros(4, 8, dtype=torch.bool) + few[:, :2] = True # fewer allowed than top-k: the surplus slot carries no weight + w3, i3 = local_eval.gate_masked(gate, x, few, torch.nn.functional.linear) + assert ((w3 > 0).sum(-1) == 2).all() and (i3[w3 > 0] < 2).all() From 199687d311ec17dd272cb5a265b96efd8e3d35d8 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:41:34 -0800 Subject: [PATCH 44/74] docs: V4.1 build plan (pinned oMLX fork, compact router, kill tests, inventory and ideas placed); C8 refuted; prior art --- .../2026-11-08-v41-reap-recovery-log.md | 24 ++ .../2026-11-10-v41-wow-build-plan.md | 285 ++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index cef4333..03132b4 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -782,3 +782,27 @@ What it says: target architecture, a critique of the approach, ten falsifiable conjectures, eleven ordered experiments, and a proposed working title conditional on C1. The first experiment (E1) needs no new fits and runs after pass 2. + +### 2026-11-09/10: C8 refuted, prior art that changes the plan, build plan written + +- 96-expert follow-up (pod, 54 sessions, domain-mean KL): one pass on full-model routing 1.123; + two passes through the total-mass 192 1.150; three passes 1.154; the Mac's pick inside REAP-50 + 1.163. Re-measuring routing made it worse by 0.027 where conjecture C8 predicted at least 0.05 + better. Same confound as before (the multi-pass tables used 12 times less calibration data); no + interval computed yet. C8 is refuted as stated. +- Prior-art search (2026-11-09): oMLX already runs the full V4.1 on a 128 GB Mac with experts on + SSD (waits on every miss, 4.2–5.6 tok/s, drafter with offload allowed since 2026-09-23); + re-routing a miss is published (Cache-prior, SMoE, MoE-CORE, SwapMoE); a nested bit-slice cache + is published (SliceMoE); 2-bit V4.1 builds exist three times. The "frozen cache" framing is not + stated anywhere found, but is too close to SwapMoE to carry a paper alone. +- Jack: a running model is the bar ("we need the decoder and the rest"); oMLX may be run and the + rule against running Rapid-MLX no longer applies. +- [Build plan](2026-11-10-v41-wow-build-plan.md): answers to the three sizing questions with + sources and dates, three rival plans and why two lost, the chosen route (a pinned oMLX fork with + a compact router that re-routes a miss at no time cost), six kill tests for days 0–1, all 40 + inventory items placed, Jack's seven ideas placed, prior art, decisions. +- Sizing facts worth keeping in view: non-expert weights are 9.85 GB as stored; a 2.03-bit expert + is 8.99 MB, so 192 per layer is 69 GB and 384 is 138 GB; the per-token read floor is 10.7–13.0 GB + (31–45 tok/s on this Mac); the "91 GB thrash" was measured on an M5 Max. +- Pod: with Jack's go, a 33-minute eval of static 250 and 288 of 384 against the full model is + running before the pod is terminated. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md new file mode 100644 index 0000000..5638999 --- /dev/null +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -0,0 +1,285 @@ +# V4.1 on a 128 GB Mac, generating: build plan + +Written 2026-11-09/10 from three read-only surveys (repo inventory, sizing, prior art), three rival +plans drafted from different premises, and a red-team of each against the code. Everything marked +"estimate" is a planner's estimate and has not been measured. Days are working days. + +**The bar.** A stranger with a 128 GB Mac runs one command and watches the full +DeepSeek-V4.1-Flash, with all 384 experts per layer reachable, do an agentic coding task at a speed +they would use. Nothing generates V4.1 in this repo today. + +**The claim to build toward.** The full model already runs on 128 GB (oMLX), at 4.2–5.6 tok/s, +because every cache miss waits for the SSD. The fully resident build (Rapid-MLX) does 28 tok/s +without the drafter and needs 256 GB. Our claim is the trade in between: a miss that costs no +time, at a measured and bounded loss of exactness. It is a speed and evidence claim, not a new +idea; section 7 says which ideas are already shipped. + +## 1. The three questions + +### 1.1 How much of a V4.1 decoder already exists here? + +Source: repo inventory, 2026-11-09, read file by file against the reference +(`~/Models/v41-reap50/inference/model.py`). + +- No V4.1 decoder. A complete MLX decoder for the older V4-Flash exists + (`src/ramp/models/deepseek_v4_flash_adapter.py`, `src/keep/quality/dsv4_mtp_runtime.py`): + prefill, KV cache, greedy decode, drafter generation with rollback. Correct on toys; never shown + fast (its one decode figure, 0.24 tok/s, was paging on an M5 Max). +- Reusable for V4.1: router and SwiGLU (`adapter:913-1078`), RoPE, sinkhorn math and its fused + Metal kernel (`src/ramp/kernels/hc_sinkhorn.py`), expert compute from 2-bit codes + (`src/ramp/kernels/e8p_m1.py`, 202–206 GB/s on this Mac at V4-Flash shapes), FP8 residents as + mxfp8 (`src/keep/convert/fp8_block.py`), native FP4 experts (`mx.gather_qmm`), `preadv` I/O + (`src/keep/quality/dsv4_teacher_runner.py:640-760`), and a numeric oracle: exact CPU ports of all + six reference kernels (`scripts/runpod/v41_reap/torch_kernel.py`). +- Missing: Engram in MLX entirely; V4.1 attention (compressed KV shared across layers, shared index + keys, two-level candidate blocks); hyper-connection wiring (mix weights used one sublayer late); + a cache layout with rollback; drafter changes; a loader; a sampler; the chat format; any server. +- The reference cannot run on a Mac as shipped (its kernels are CUDA-only) and decodes one token + per call after the prompt, so chunked prefill and drafter verification have no reference to copy. +- Outside this repo the whole decoder exists twice, under permissive licences (section 7). + +### 1.2 Where does decode time go? + +Source: sizing survey, 2026-11-09, from the safetensors headers and the reference; no V4.1 decode +has been timed on any Mac in this repo. + +| Read per decoded token, full model | GB | +|---|---:| +| Attention | 5.15 | +| Shared experts | 1.42 | +| Output head | 1.32 | +| Router, hyper-connections, norms, Engram projection | 0.63 | +| **Non-expert subtotal** | **8.52** | +| 240 routed experts at native FP4 / at 2.03 bits | 4.51 / 2.16 | +| **Total** | **13.0 / 10.7** | + +- Ceiling at this Mac's measured read rate (409 GB/s on 2026-10-06, 474–477 on 2026-11-05): + 31–45 tok/s. Routed experts are 35% of bytes at native precision, not the 11% seen on V4-Flash + in August; that lead does not transfer. +- With the drafter, one verify pass serves several tokens, so non-expert reads are shared and + experts become the larger share. Break-even is about 2.2 accepted tokens per cycle (estimate); + Rapid-MLX reports about 3.1. +- oMLX's 4–6 tok/s is consistent with waiting on about 74 expert loads per token (estimate from + its hit rates). Removing the waits alone caps near 20 tok/s; the rest depends on staying on its + fused kernels. +- Unknown until measured here: the time split by component, sinkhorn cost (80 calls per token), + the indexer scan at long context. + +### 1.3 What is resident besides routed experts? + +Source: sizing survey, 2026-11-09. + +| Item | GB | +|---|---:| +| All non-expert weights as stored (FP8) | 9.85 | +| The same expanded to bf16 | 16.7 | +| Drafter as stored | 7.9 | +| Context state at 64K tokens (3,200 bytes per token) | 0.21 | +| One routed expert: native FP4 / our 2.03-bit code / oMLX 3-bit | 0.0188 / 0.0090 / about 0.0176 | +| 192 experts per layer at 2.03 bits | 69 | +| 384 experts per layer at 2.03 bits | 138 | + +- Memory goes almost entirely to routed experts. The Engram page cache is the open term: up to + 0.77 MB per new token, so as much as 50 GB at 64K if n-grams never repeat (upper bound). +- The "91 GB thrash" was measured on an M5 Max 128 GB on 2026-08-21, not on this M4 Max. Nobody + has measured the ceiling here. +- At the 80 GB process cap about 160 experts per layer fit; about 250 need roughly 108 GB with + residents and the drafter, which one planner puts at Metal's limit on this machine. + The top 192 / 250 / 288 experts by routed mass cover 92.5% / 95.2% / 97.5% of routed mass. + +## 2. Chosen architecture + +**Build on oMLX as a pinned fork; plug in our cache policy, selection tables and (if it passes its +kill test) our 2-bit kernels.** + +- oMLX (`~/Development/_refs/omlx`, commit `4213c328`, Apache-2.0 with MIT parts) already has the + V4.1 decoder, the Engram reader, the drafter, an expert store with `pread` into slots, a server + with tool-call parsing, and it reads the FP4/FP8 checkpoint already on this Mac. +- The change is a **compact router**: each layer keeps C experts in slot stacks and the router's + weight and bias are sliced to those C rows. oMLX's fused kernels then pick the top 6 among + residents and renormalise. That is the re-route, with no wait, no read and no extra sync. Demand + for absent experts is read once per token from the full router and drives background admission. +- New code (estimate, about 300 lines plus four edits): `routed_cache.py` beside + `omlx/patches/deepseek_v41/moe_offload.py`; later `e8p_moe.py` calling + `src/ramp/kernels/e8p_m1.py`. +- Stock oMLX in its exact, waiting mode is both the day-1 measurement tool and the baseline. + +Why the rivals lost: + +- **Extend our own V4-Flash runtime.** Its own planner called it "a rewrite, and a fourth + implementation": the adapter differs from V4.1 in attention, hyper-connection wiring, caches + and drafter, and has no Engram. Twelve days to a first MLX token and about 47 to a demo, spent + mostly on what two projects already ship, on a runtime with no clean speed measurement. Its + E8P kernels and oracle survive as plug-ins. +- **Port the reference directly.** A torch decoder on CPU gives a first token in 3.5 days at + 20–80 s per token, and none of it carries into a fast runtime. Its one unique product, a + generation oracle, is covered by stock oMLX from day 1 at about 5 tok/s. Kept from it: parity + gates must be statistical (rounding alone moves KL by 0.1–0.3 per position between runs). +- **Rapid-MLX as the base.** The cleaner kernel host (9.4k lines against about 60k) and the faster + resident path, but it has no expert store, refuses to start under 224 GiB, and needs a download + of about 320 GB. It is the fallback if oMLX's fused path proves slow. + +Red-team of the chosen plan: + +- **Speed is unverified until day 4.** oMLX publishes no fully resident V4.1 number. The + planner's estimate is 15–20 tok/s without the drafter. +- **Their codebase moves** (40 commits to the V4.1 directory since October; exact `mlx` pin; + compiled kernels). Pin the commit and do not rebase before the demo. +- **Little of the system is ours.** What is: the zero-latency miss on a Mac, the policies in + section 5, a denser code, and the measurements. +- **oMLX chose exactness on purpose**, and cites a result that pinning the hot half and zeroing the + rest costs about 91% accuracy on a maths benchmark (source could not be verified; zeroing is not + re-routing). Teacher-forced KL will not settle this. Generation has to. + +## 3. Kill tests before any building (days 0–1) + +| # | Test | Kills | +|---|---|---| +| K1 | Stock oMLX on this Mac, local pruned checkpoint: tok/s, hit rate, resident memory, and a replayed prompt (hit rate near 1) for its no-miss floor. | The base, if the no-miss floor is under about 10 tok/s: switch to Rapid-MLX's decode kernels. | +| K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. | Our lattice kernel in the runtime: ship affine 2-bit instead. | +| K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod (running). | Nothing; it sets how much the cache has to recover and the target C. | +| K4 | E1: expert cache with re-route against static sets, teacher-forced (queued tonight). | The quality case for the cache, if it is not better than a static set. | +| K5 | 20 lines: an MLX array over a 1 GB memory-mapped file through the E8P kernel; check footprint and `mincore`. | Idea 3 (page cache), if it copies. | +| K6 | Offline, from stored routes: how well token ids predict experts; how well a prompt's demand predicts the rest of the session; recall of the drafter's 256-d head. | Ideas 2, 6 and 7 respectively. | + +## 4. Critical path + +**To the first generated token on this Mac: day 1.** Stock oMLX on `~/Models/v41-reap50` (already +on disk, no download). That token comes from the pruned 192-expert model in exact mode. It is a +measurement, not the result. + +**To the first no-wait token: day 4** (estimate; I would plan 4–6). + +| Day | Work | Done when | +|---|---|---| +| 1 | K1, K2, K5; parity of oMLX against `torch_kernel.py` on the local model | Top-1 match on 200 positions and KL under 0.01 against the oracle | +| 2–4 | Compact router with a static resident set | Bit-identical to stock with C = all; all 40 layers stay on the fused path; at least 12 tok/s without the drafter | + +**To the recorded demo: about 19 days with no slack; plan 19–26.** + +| Days | Work | +|---|---| +| 5–7 | Background admission, warm tables from our selection, drafter on | +| in parallel | Pod or download: the other 192 experts and the full router rows (decision 2) | +| 8–11 | Lattice-code expert path and the pool format (only if K2 passed) | +| 12–13 | The 384 pool end to end | +| 14–16 | Generation benchmarks, baseline (stock oMLX at equal memory), two ablations | +| 17–19 | Wrapper script, model card, recording | + +## 5. The seven ideas, placed + +None is on the path to the first token. Ideas 4 and 5 share one mechanism. Two planners placed +them independently; where they disagreed the more sceptical reading is kept. + +| # | Idea | Attaches at (oMLX fork) | Cheapest kill | When | +|---|---|---|---|---| +| 1 | Drafter with a no-wait cache | `mtp.py` and the verify path in `decode_fused.py`; admission only between draft-verify cycles so one cycle sees one resident set. Needs a multi-row lattice kernel. | Day 7: tokens per cycle over cycle time under 1.5× single-token decode. | Demo | +| 2 | Drafted tokens as lookahead | Admission queue in `routed_cache.py`; needs a token-id → expert prior. | K6: recall of the next four tokens' experts against plain decayed counts. | After | +| 3 | OS page cache as the cache | A native module with a no-copy Metal buffer over a mapping. | K5, as a standalone Metal test: dead if Metal wires the pages or a fault stalls the GPU. | After | +| 4 | Exactness inside tool calls | `output_parser.py` already tracks the tool-call state; the exact path is the existing load-and-wait. Drafter depth 1 inside a call. | 50 tool-call prompts at C = 160: cut if re-routed validity equals exact. | Demo, conditional (1–2 d) | +| 5 | Exact-to-fast dial | The routing weight that missed, scored in `routed_cache.py`; same exact path as 4. | Offline now: the harness already takes a threshold, so plot KL against the share of tokens that wait. A flat curve kills the runtime dial. | Curve before the demo; runtime after | +| 6 | Prompt as profile | Prefill branch of the MoE call: accumulate the full router's weight per expert, bulk-load before decode. | K6: mass covered over the next 64 generated tokens, against the static set plus counts. | Demo if it passes (1 d) | +| 7 | Shortlist head | `head.py` plus the drafter's 256-d head. | Day-4 profile: cut if the head is under 5% of token time. | After | + +What the code says about them: + +- **Idea 1 is the strongest, with a measurement behind it.** oMLX has allowed the drafter with + offload since 2026-09-23, and it gains only 29% (6.6 → 8.5 tok/s at 81–89% acceptance), because a + four-token verify asks for four times the experts and waits for them (333 ms against 151 ms). + That wait is exactly what a no-wait cache removes. +- **Idea 3 is harder than it looked.** Neither runtime computes on mapped pages; both copy into + allocated buffers, and Rapid-MLX's loader warns that a cold page-in inside a GPU command trips + Metal's watchdog. A safe partial version: use `mincore` only to rank absent experts that are + already in the page cache, so they are admitted first. +- **Ideas 2 and 7 are weak as stated.** A verify pass already computes the drafted tokens, so + Engram lookahead saves under 1% of wall time, and oMLX's own router prefetch ran 2–15% slower. + The drafter's 256-d head is a bias on the previous token and never sees the hidden state, so a + shortlist would need a newly trained projection. + +## 6. The inventory, placed + +Exists = in this repo. oMLX = reuse from the fork (Apache-2.0/MIT). Build = new work. Cut = not for +the demo, with the reason. + +| # | Item | Placement | +|---|---|---| +| 1 | Block port | oMLX `language.py` | +| 2 | Engram | oMLX `engram.py`, `storage.py` | +| 3 | FP8/FP4 loaders | oMLX `loading.py`, `convert.py` | +| 4 | Incremental decode, KV, prefill | oMLX `decode_fused.py`, `cache.py` | +| 5 | Tokenizer, chat, tool calls, sampling | oMLX `encoding.py`, `tool_parser.py` | +| 6 | Drafter | oMLX `dspark.py`, `mtp.py`; build the verify check on the compact router | +| 7 | Parity harness | Build on `torch_kernel.py`, `local_parity.py`; statistical gates | +| 8 | Speed: experts from 2-bit codes, attention kernels | Build `e8p_moe.py` on `e8p_m1.py` if K2 passes; attention kernels from oMLX | +| 9 | On-disk pool format | Build: stacked safetensors readable by oMLX's row reads | +| 10 | Resident-set manager | Build on oMLX `_ExpertSlots` | +| 11 | Admission path | Build: background, applied between tokens | +| 12 | Router changes | Build: compact router. Gate floor cut until measured in the runtime | +| 13 | Prefill behaviour | Comes with the compact router | +| 14 | Memory-pressure response | oMLX's memory guard; shrink C | +| 15 | I/O sharing with Engram | oMLX: the expert pool is already separate | +| 16 | Warm start, persistence | Build: save and load slot ids | +| 17 | Hot tier above 2 bits | Cut: unproven, and one tier is enough for the demo | +| 18 | 2-bit codes for all 384 | Build: fit on the Mac after fetching the other 192 experts, or on a pod (decision 2) | +| 19 | Importance statistics, router rows | Exists for all 384 (statistics); router rows from the pod volume | +| 20 | Non-expert weight format | oMLX: source FP8 as stored (9.85 GB) | +| 21 | Selection and warm tables | Exists (`rerank.py`, total mass); regenerate for the pool | +| 22 | Packaging, download | Build after the demo: codes pool on Hugging Face (public: decision 4) | +| 23 | Licence review | Half a day: MIT model, Apache-2.0 fork notice | +| 24 | Eval set version 2 | Build (paper track, not the demo) | +| 25 | Generation benchmarks | Build: tool-call validity, code tasks that execute, one agentic task | +| 26 | Baselines | Stock oMLX at equal memory, run here; Rapid-MLX numbers quoted (216 GB does not fit) | +| 27 | Hardware measurements | Build: decode ms per token by component, memory, SSD rate, swap count | +| 28 | Smaller-RAM emulation | Cut until after the demo; the slot count gives it later | +| 29 | Ablations | Two: static set against cache, wait against re-route | +| 30 | Second model | Cut from the demo; stays in the paper plan | +| 31 | Reproducibility | Pinned fork commit, checksums, one script | +| 32 | CLI and server | oMLX `omlx serve` | +| 33 | One-command install | Build: wrapper script | +| 34 | Support matrix, CI | Cut to "128 GB M-series" | +| 35 | Recorded demo | Build | +| 36 | Cards, paper | Model card for the demo; paper separately | +| 37 | Gate: cache beats static | K4, tonight | +| 38 | Gate: cache over 384 recovers the pruning loss | Teacher-forced in our harness once item 18 exists; then in generation | +| 39 | Gate: speed bound | K1, then the day-4 measurement | +| 40 | Gate: compute from 2-bit codes fast enough | K2 | + +## 7. Prior art + +| Project or paper | What it does | Reuse | Licence | +|---|---|---|---| +| DeepSeek V4.1-Flash | Model, torch reference, Engram, drafter | Reference and oracle | MIT, weights included | +| oMLX | V4.1 on MLX; expert store on SSD, waits on a miss; drafter with offload; 4.2–5.6 tok/s on 128 GB | The base | Apache-2.0, MIT parts | +| Rapid-MLX | V4.1 fully resident: 216 GB, KL 0.26, 28 tok/s plain, 33–49 with the drafter; Engram prefetch | Fallback kernels | Apache-2.0 | +| ds4 | C/Metal engine with a slab expert cache; waits on a miss | Design reading | MIT | +| llama.cpp | V4.1 merged 2026-10-25; IQ2_XXS is an E8-lattice 2-bit format | — | MIT | +| Cache-prior (2412.00099), SMoE (2508.18983), MoE-CORE (2610.01950) | Re-route or substitute a miss using router scores | The rule | papers | +| SwapMoE (2308.15030) | Dynamic resident set against static pruning at equal memory, small models | The comparison | paper | +| SliceMoE (2512.12990), MatMoE (2611.01704) | Nested bit slices; nested codes chosen by routing score | — | papers | +| HOBBIT, DynaExq, DyMoE, Fate, AdapMoE, MoE-Infinity | Low-precision fallback, asynchronous promotion, prefetch | Baselines | papers | +| REAP (2510.13999) | Per-route pruning score; deliberately avoids frequency weighting | Already implemented here | Apache-2.0 | + +Which of our ideas someone has already shipped: + +- **Shipped:** the full model on 128 GB with experts on SSD (oMLX, ds4); drafter with offload + (oMLX); 2-bit V4.1 builds (three of them); Engram on SSD with prefetch (Rapid-MLX). +- **Published, not shipped on a Mac:** re-routing a miss; nested precision caches; a dynamic set + against static pruning. +- **Not found:** a miss that costs no time in a Mac runtime at this scale; exactness switched by + grammar state; the exact-to-fast curve; a residency check on the page cache with re-route; + drafted tokens as lookahead; quality against pruning at V4 scale; selection by total routed mass. + +## 8. Decisions for Jack + +1. **Memory cap.** 80 GB holds about 160 experts per layer. About 250 needs roughly 105–110 GB, + which is above where the M5 Max thrashed. +2. **Source of the other 192 experts.** (a) Fetch their FP4 weights from the pod volume (about + 144 GiB) and fit our codes on the Mac overnight; (b) fit on a pod (spend; the fit has only run + on the Mac GPU); (c) download oMLX's 3-bit build (about 331 GiB), which gives a 384-expert + token soonest. Free disk is about 430 GiB, so not all of them. +3. **Fork.** Whether a pinned oMLX fork is acceptable as the shipped form, and whether it is public. +4. **Claim.** A model that is not exact is the product. The demo should not be published before + the generation check. +5. **Housekeeping.** A planner's Python call created `~/Development/_refs/omlx/.venv` (1.8 GB), + `uv.lock` and an egg-info directory in the oMLX clone. Remove or keep. +6. **Paper title.** Still open; this plan does not depend on it. From 027dacea4097190707e82f49c6007126ddd6e387 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:45:20 -0800 Subject: [PATCH 45/74] build plan: memory cap stays at 80 GB (C = 160 on the Mac) --- docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 5638999..c140b69 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -271,8 +271,10 @@ Which of our ideas someone has already shipped: ## 8. Decisions for Jack -1. **Memory cap.** 80 GB holds about 160 experts per layer. About 250 needs roughly 105–110 GB, - which is above where the M5 Max thrashed. +1. **Memory cap. Decided 2026-11-09: stay at 80 GB for now** (the Mac is under load from other + sessions; it can be raised later, and pods are the route for anything that needs more). The + build therefore targets C = 160 experts per layer on the Mac. Anything that needs about 250 + resident (roughly 108 GB) is measured teacher-forced on a pod until the cap changes. 2. **Source of the other 192 experts.** (a) Fetch their FP4 weights from the pod volume (about 144 GiB) and fit our codes on the Mac overnight; (b) fit on a pod (spend; the fit has only run on the Mac GPU); (c) download oMLX's 3-bit build (about 331 GiB), which gives a 384-expert From dfe8f7561d37fa1fe932a87bfdcb28f36971bf66 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:54:50 -0800 Subject: [PATCH 46/74] build plan: K1 runbook for stock oMLX --- .../2026-11-10-v41-wow-build-plan.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index c140b69..3dbfa88 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -285,3 +285,32 @@ Which of our ideas someone has already shipped: 5. **Housekeeping.** A planner's Python call created `~/Development/_refs/omlx/.venv` (1.8 GB), `uv.lock` and an egg-info directory in the oMLX clone. Remove or keep. 6. **Paper title.** Still open; this plan does not depend on it. + +## 9. Runbook for K1 (first generated token, stock oMLX) + +Checked 2026-11-09 by reading the clone and running only `omlx --help`: the environment at +`~/Development/_refs/omlx/.venv` imports (`mlx` 0.32.4, `omlx` 0.7.4.dev1). Not yet done: the +native kernels are not built, and no model has been loaded. + +Run when no other heavy job is on the Mac (E1 holds it tonight): + +1. `make kernels` in the clone (oMLX's fused paths need them; a plain install skips them). +2. A model directory containing a link to `~/Models/v41-reap50`. oMLX's V4.1 loader accepts the + original checkpoint format; whether it accepts a 192-expert count is the first thing this run + tests. +3. Model settings: `moe_expert_offload_enabled`, `moe_expert_offload_resident_fraction` 0.375 + (72 of 192 experts at native FP4, about 54 GB), `deepseek_v41_engram_ssd_offload`; memory guard + below 80 GB. oMLX's own figures show peak footprint about 12 GiB above its Metal-active size, so + start lower (0.25) if the first load comes close to the cap. +4. `omlx serve --model-dir …`, then two 64-token chat completions on one prompt (cold, then warm) + and the same prompt replayed for the no-miss floor. +5. Record: decode tok/s, prefill tok/s, hit rate (oMLX logs it per request), peak footprint, and + whether every layer stayed on the fused path. + +Reference points from oMLX's own document: on a smaller MoE model it measures 122.5 tok/s fully +resident, 59.4 at 50% residency and 40.1 at 25%, so waiting on misses costs it a factor of two to +three there; on V4.1 it says decode "is bound by miss latency at one to two misses per layer per +token". + +Not runnable yet: the offline checks for ideas 2 and 6 (K6) need per-token router demand, which +the harness does not dump; the stored routes are a 2,048-token sample without token ids. From 780721f9b44047d00de81f4900751e09cecdec85 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:20:32 -0800 Subject: [PATCH 47/74] docs: static resident-set costs at 250 and 288 (K3), pod terminated --- .../2026-11-08-v41-reap-recovery-log.md | 23 +++++++++++++++++++ .../2026-11-10-v41-wow-build-plan.md | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 03132b4..e41684f 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -806,3 +806,26 @@ What it says: (31–45 tok/s on this Mac); the "91 GB thrash" was measured on an M5 Max. - Pod: with Jack's go, a 33-minute eval of static 250 and 288 of 384 against the full model is running before the pod is terminated. + +### 2026-11-10: what a fixed resident set costs (K3), pod terminated + +- Static sets by total routed mass against the full model, native precision, no cache, 54 eval + sessions, domain-mean KL (pooled in brackets) and top-1 agreement: + + | Experts per layer | Routed mass kept | KL | Top-1 | + |---:|---:|---:|---:| + | 96 | 74.0% | 1.123 (1.223) | 0.709 | + | 192 | 92.5% | 0.532 (0.650) | 0.814 | + | 250 | 95.2% | 0.323 (0.449) | 0.860 | + | 288 | 97.5% | 0.235 (0.357) | 0.887 | + + Tool-use and the two agentic domains carry most of what remains (tool-use 1.058 at 250 and + 0.837 at 288; code-diffs 0.045 and 0.021). Mass kept is a poor predictor of KL: the last 2.5% of + mass still costs 0.235. +- For the build plan: at the 80 GB cap about 160 experts fit, where a fixed set costs roughly + 0.6–0.7 KL by interpolation (not measured); at 250 it costs 0.32. The cap, more than the code + density, sets how much the cache has to recover. +- H100 pod `q8vcl6ob0hf0lk` terminated 06:21 UTC after about 5.3 h ($21). Volume `yqazs7aqe6` + (720 GB) keeps the full model and all eval outputs (`/workspace/x1/eval`, `eval2`, `eval3`). +- Mac: the last sweep on the 1-bit ladder was stopped at Jack's request (17 CPU-hours in, results + not kept). E1 was cut from eight variants to four and is running. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 3dbfa88..9c894e0 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -137,7 +137,7 @@ Red-team of the chosen plan: |---|---|---| | K1 | Stock oMLX on this Mac, local pruned checkpoint: tok/s, hit rate, resident memory, and a replayed prompt (hit rate near 1) for its no-miss floor. | The base, if the no-miss floor is under about 10 tok/s: switch to Rapid-MLX's decode kernels. | | K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. | Our lattice kernel in the runtime: ship affine 2-bit instead. | -| K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod (running). | Nothing; it sets how much the cache has to recover and the target C. | +| K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod. **Done 2026-11-10:** KL 0.323 at 250 and 0.235 at 288 (0.532 at 192, 1.123 at 96), native precision, no cache. | Nothing; it sets how much the cache has to recover. At the 80 GB cap (about 160 experts) a fixed set costs roughly 0.6–0.7 by interpolation, so the cap matters more than the code density. | | K4 | E1: expert cache with re-route against static sets, teacher-forced (queued tonight). | The quality case for the cache, if it is not better than a static set. | | K5 | 20 lines: an MLX array over a 1 GB memory-mapped file through the E8P kernel; check footprint and `mincore`. | Idea 3 (page cache), if it copies. | | K6 | Offline, from stored routes: how well token ids predict experts; how well a prompt's demand predicts the rest of the session; recall of the drafter's 256-d head. | Ideas 2, 6 and 7 respectively. | From 99f983f39058030b3255c87eba088bac3b51c11c Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:05:19 -0800 Subject: [PATCH 48/74] docs: E1 result, an expert cache with re-route beats a static set at equal memory (C1 holds in the pool of 192) --- .../2026-11-08-v41-reap-recovery-log.md | 49 ++++++++++++++++++- .../2026-11-09-routed-precision-plan.md | 4 +- .../2026-11-10-v41-wow-build-plan.md | 4 +- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index e41684f..7a7b97d 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -598,13 +598,13 @@ to 100; partial credit only for measured progress). | 1 | Nested codes, all layers, validated | 10 | 10 | fit 40/40; error 0.102 gate/up, 0.084 down; end to end +0.053 KL against flat 2-bit | | 2 | Per-token routed tiers, implemented and tested | 10 | 10 | `--dynamic/--tau/--tau-drop`, smoke-tested | | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | -| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 0 | pass 1: R beats S by 0.068, but loses to 144 flat 2-bit experts by 0.025; D pending (pass 2b) | +| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 12 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. The rest needs the pool of 384 and the pod confirmation (E7) | | 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | | 6 | Real Apple Silicon streaming measured, not simulated | 10 | 0 | | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **31** | | +| | **Total** | 100 | **43** | | ### 2026-11-09: prior art, plan, restart @@ -829,3 +829,48 @@ What it says: (720 GB) keeps the full model and all eval outputs (`/workspace/x1/eval`, `eval2`, `eval3`). - Mac: the last sweep on the 1-bit ladder was stopped at Jack's request (17 CPU-hours in, results not kept). E1 was cut from eight variants to four and is running. + +### 2026-11-10: E1, an expert cache against a fixed set (C1 holds inside the pool of 192) + +- Setup: REAP-50 pool (192 experts per layer), every resident expert at 2 bits, no low-bit copy. + A token routes among the experts resident when it arrives (`--miss reroute`); experts it asked + for above gate 0.2 are admitted afterwards and serve from the next token on. Warm start from the + static top-N. 22 sessions, first 1,024 tokens, domain-mean KL against the full model, paired + bootstrap over sessions. + + | Variant | Residents | KL | Top-1 | Reads at 30 tok/s | + |---|---:|---:|---:|---:| + | static (`rs144`) | 144 | 0.812 | 0.770 | 0 | + | LRU cache (`rr144`) | 144 | 0.756 | 0.785 | 0.45 GB/s | + | static (`prunedmass-96`, step 2c) | 96 | 1.039 | 0.733 | 0 | + | LRU cache (`rr96`) | 96 | 0.809 | 0.777 | 1.72 GB/s | + | per-session oracle set (`ro96`) | 96 | 0.789 | 0.782 | 0 | + +- Path check: `rs144` reproduces the earlier pruned 144 exactly (0.8123 against 0.812), so + masked routing in the new path equals pruning. +- Cache against static at equal memory: −0.056 [−0.066, −0.046] at 144; −0.231 [−0.248, −0.211] + at 96. A cache of 96 equals a static 144 (−0.004 [−0.020, +0.008]): the same quality for two + thirds of the expert memory. A cache of 144 is within 0.009 [−0.001, +0.012] of all 192 + resident (0.747). +- C1 passes its own test (gain above 0.02 at both sizes, intervals clear of zero, reads under + 2.5 GB/s). Not checked: the "confident positions" form of the gain, because the Mac eval files + do not store the teacher's NLL. +- C5: the oracle beats the LRU by 0.020 [0.005, 0.027], above the predicted 0.015. All of it is + warm-up: the gap is 0.050 in the first 256 tokens and 0.002–0.017 after. So admission policy is + near its ceiling, and the one thing worth building is a better starting set (idea 6, the prompt + as a demand profile). +- C6 holds at 144 (0.45 GB/s; 1.6 admissions per token, 15 MB) and misses its 1 GB/s line at 96 + (1.72 GB/s; 6.2 admissions per token, 57 MB). +- C4 untested: the drop variant was cut from the run. +- Where the gain is: english-chat (0.760 to 0.468 at 144; 1.195 to 0.497 at 96), business (0.971 + to 0.763; 1.391 to 0.786). Agentic and tool-use text does not move (tool-use 3.27 to 3.33 at + 144). The static set was chosen on the mass of the whole calibration mix, so the cache helps + where a session differs from the mix; tool-use loss comes from the pool of 192 itself and needs + the pool of 384 (C2). +- The gain is flat along the session (−0.268, −0.233, −0.209, −0.212 by quarter at 96). +- Limits: pool of 192 and not 384; first 1,024 tokens of 22 sessions; reads are counted, not + measured; a fetch is assumed to land within one token. The log's `admissions_per_token_layer` + is divided by the layer count twice; `routed_report.py` and the counts above are right. +- Rubric item 4 moves from 0 to 12 of 25. Total 43. +- Run cost: 6,757 s forward, peak RSS 62.4 GB. Nothing is running on the Mac or on Runpod. + diff --git a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md index 903ee86..8686c52 100644 --- a/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md +++ b/docs/deepseek-v4-flash/2026-11-09-routed-precision-plan.md @@ -11,7 +11,7 @@ Pruning is a cache that never updates. Keep every resident expert at a usable pr let the set of resident experts follow the session, and send a token whose expert is absent to the best resident one instead of to a worse copy. Precision above 2 bits is a second, smaller question. -This is a conjecture (C1 below), not a result. What is measured so far: a routed cache of +C1 held in its first test (E1, 2026-11-10, inside the pool of 192): an expert cache with re-route beats the static set by 0.056 KL at 144 residents and 0.231 at 96, and a cache of 96 equals a static 144. C2 (the pool of 384) is the open half. Measured before that: a routed cache of residuals beats a fixed one by 0.068 KL; a 1-bit fallback is the dominant cost; dropping low-gate slots is free; the second bit is worth more than breadth. @@ -110,7 +110,7 @@ C1, C2 and C6 carry the title. C7–C8 are a second, independent result. | # | What | Where, cost | Gate | |---|---|---|---| | E0 | Pass 2 on the old ladder (oracle gap for C5). | Mac, finishing about 23:00. | — | -| E1 | Expert cache at 2 bits inside REAP-50 (`local_eval.py --miss drop\|reroute`, built and unit-tested). Eight variants: static, LRU (τ 0.2 and 0.1), oracle and drop at N = 144; static, LRU and oracle at N = 96. The static variants use exactly the sets scored earlier, so they must reproduce 0.812 and 1.039: that is the end-to-end check of the new path. Queued (`jobs/e1.sh`) behind E0 with a 64-token smoke test. Stratified columns in `routed_report.py` are still to build. | Mac, about 3.5 h. | Decides C1, C4, C5. | +| E1 | **Done 2026-11-10** (trimmed to four variants). Expert cache at 2 bits inside REAP-50 with re-route on a miss: 0.756 against 0.812 static at 144; 0.809 against 1.039 at 96; oracle 0.789 at 96; the static control reproduced 0.812. Details in the recovery log. | Mac, 1.9 h. | C1 holds; C5 gap 0.020, all warm-up; C6 holds at 144, misses at 96; C4 not run. | | E2 | 96 experts, one pass against two. | Pod, running. | C8. | | WR | Export and download the total-mass 192: the honest static baseline and a better pool of 192. | CPU pod, overnight. | — | | E3 | If C1 holds: 2-bit codes for all 384 experts and the full router rows, as a codes-only pool (about 150 GB). Size the fit first; it has only run on the Mac GPU. | Pod. | Needs C1. | diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 9c894e0..ff74749 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -138,7 +138,7 @@ Red-team of the chosen plan: | K1 | Stock oMLX on this Mac, local pruned checkpoint: tok/s, hit rate, resident memory, and a replayed prompt (hit rate near 1) for its no-miss floor. | The base, if the no-miss floor is under about 10 tok/s: switch to Rapid-MLX's decode kernels. | | K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. | Our lattice kernel in the runtime: ship affine 2-bit instead. | | K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod. **Done 2026-11-10:** KL 0.323 at 250 and 0.235 at 288 (0.532 at 192, 1.123 at 96), native precision, no cache. | Nothing; it sets how much the cache has to recover. At the 80 GB cap (about 160 experts) a fixed set costs roughly 0.6–0.7 by interpolation, so the cap matters more than the code density. | -| K4 | E1: expert cache with re-route against static sets, teacher-forced (queued tonight). | The quality case for the cache, if it is not better than a static set. | +| K4 | E1: expert cache with re-route against static sets, teacher-forced. **Passed 2026-11-10:** the cache beats the static set by 0.056 KL at 144 residents and 0.231 at 96 (pool of 192, 2 bits); a cache of 96 equals a static 144; the oracle gap (0.020) is all in the first 256 tokens. | The quality case for the cache, if it is not better than a static set. | | K5 | 20 lines: an MLX array over a 1 GB memory-mapped file through the E8P kernel; check footprint and `mincore`. | Idea 3 (page cache), if it copies. | | K6 | Offline, from stored routes: how well token ids predict experts; how well a prompt's demand predicts the rest of the session; recall of the drafter's 256-d head. | Ideas 2, 6 and 7 respectively. | @@ -239,7 +239,7 @@ the demo, with the reason. | 34 | Support matrix, CI | Cut to "128 GB M-series" | | 35 | Recorded demo | Build | | 36 | Cards, paper | Model card for the demo; paper separately | -| 37 | Gate: cache beats static | K4, tonight | +| 37 | Gate: cache beats static | K4, passed 2026-11-10 | | 38 | Gate: cache over 384 recovers the pruning loss | Teacher-forced in our harness once item 18 exists; then in generation | | 39 | Gate: speed bound | K1, then the day-4 measurement | | 40 | Gate: compute from 2-bit codes fast enough | K2 | From 681e131b940253c46bf8665dee8e13b53a7c5146 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:03:26 -0800 Subject: [PATCH 49/74] K1: stock oMLX on the pruned checkpoint; a fixed resident set with a sliced router decodes at 30 tok/s --- .../2026-11-08-v41-reap-recovery-log.md | 16 +++ .../2026-11-10-v41-wow-build-plan.md | 36 +++++- scripts/v41_build/omlx_compact.py | 105 ++++++++++++++++++ scripts/v41_build/omlx_counters.py | 41 +++++++ 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 scripts/v41_build/omlx_compact.py create mode 100644 scripts/v41_build/omlx_counters.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 7a7b97d..554fa5d 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -874,3 +874,19 @@ What it says: - Rubric item 4 moves from 0 to 12 of 25. Total 43. - Run cost: 6,757 s forward, peak RSS 62.4 GB. Nothing is running on the Mac or on Runpod. +### 2026-11-10: K1, first generated tokens on the Mac + +- Stock oMLX (pinned clone, kernels built) loads the 192-expert REAP-50 checkpoint as is. With + its expert offload at 48 of 192 experts it decodes at 3.3–4.5 tok/s, 78% hit rate, about 0.9 GB + read per token, 45.4 GiB after load. +- A fixed 48 experts per layer (top routed mass) with the router cut to those rows decodes at + 30–32 tok/s and prefills at 150 tok/s in the same memory; 64 experts run at the same speed in + 56.6 GiB. No checkpoint copy: a linked model directory plus read-side patches + (`scripts/v41_build/omlx_compact.py`). +- oMLX's offload does not use its fused decode kernels; a fixed set does. The cache will swap + rows of a stacked array in place. +- Text at 48 and 64 experts is degraded (64 loops in a long greedy completion), in line with the + static KL numbers. +- Full table and notes: the build plan's K1 result. The server is stopped; nothing is running. +- Rubric unchanged at 43: item 6 needs our own cache streaming on the Mac, not oMLX's. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index ff74749..02bfd97 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -135,7 +135,7 @@ Red-team of the chosen plan: | # | Test | Kills | |---|---|---| -| K1 | Stock oMLX on this Mac, local pruned checkpoint: tok/s, hit rate, resident memory, and a replayed prompt (hit rate near 1) for its no-miss floor. | The base, if the no-miss floor is under about 10 tok/s: switch to Rapid-MLX's decode kernels. | +| K1 | Stock oMLX on this Mac, local pruned checkpoint: tok/s, hit rate, resident memory, and its no-miss floor. **Passed 2026-11-10:** the loader takes the 192-expert checkpoint; with oMLX's own offload at 48 of 192 experts it decodes at 3.3–4.5 tok/s (hit rate 78%, about 0.9 GB read per token) and is not on the fused path; with a fixed 48 or 64 experts and a sliced router it decodes at 30–32 tok/s in the same memory. | The base, if the no-miss floor is under about 10 tok/s: switch to Rapid-MLX's decode kernels. | | K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. | Our lattice kernel in the runtime: ship affine 2-bit instead. | | K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod. **Done 2026-11-10:** KL 0.323 at 250 and 0.235 at 288 (0.532 at 192, 1.123 at 96), native precision, no cache. | Nothing; it sets how much the cache has to recover. At the 80 GB cap (about 160 experts) a fixed set costs roughly 0.6–0.7 by interpolation, so the cap matters more than the code density. | | K4 | E1: expert cache with re-route against static sets, teacher-forced. **Passed 2026-11-10:** the cache beats the static set by 0.056 KL at 144 residents and 0.231 at 96 (pool of 192, 2 bits); a cache of 96 equals a static 144; the oracle gap (0.020) is all in the first 256 tokens. | The quality case for the cache, if it is not better than a static set. | @@ -314,3 +314,37 @@ token". Not runnable yet: the offline checks for ideas 2 and 6 (K6) need per-token router demand, which the harness does not dump; the stored routes are a 2,048-token sample without token ids. + +### K1 result (2026-11-10) + +Run on the M4 Max from the pinned clone, native FP4 experts, Engram on SSD, greedy decoding, one +request at a time. Scripts: `scripts/v41_build/omlx_counters.py` (reads oMLX's expert-slot +counters, which its server does not log for V4.1) and `scripts/v41_build/omlx_compact.py` (a +fixed resident set through a linked model directory and three read-side patches; the clone is not +modified). + +| Configuration | Resident experts | Decode tok/s | Prefill tok/s | Memory after load (peak) | +|---|---:|---:|---:|---:| +| oMLX expert offload, waits on a miss | 48 of 192 | 3.3–4.5 | 6 | 45.4 GiB (54) | +| fixed set, sliced router | 48 | 30–32 | 150 | 45.4 GiB (55) | +| fixed set, sliced router | 64 | 30–31 | 145 | 56.6 GiB (67) | + +- oMLX's offload path: 78% hit rate on a 256-token completion, about 47 misses per token at + 18.8 MB each, so roughly 0.9 GB read per token; an exact replay of a 45-token exchange still + misses 39% of lookups at 48 slots. Its offloaded experts are slot wrappers that the fused decode + kernels do not accept (`decode_moe.supported`), so this path is slow even on a hit. +- The fixed set runs on the fused kernels because the experts are plain stacked arrays and + `n_routed_experts` is the resident count. This is the compact router's static case, and it is + seven times the speed of waiting on a miss at equal memory. The estimate of 15–20 tok/s was low. +- Consequence for the cache: keep the experts as one stacked array of C rows per projection and + overwrite a row in place on admission, with the router row rewritten in the same step. Do not + build on oMLX's slot wrapper. +- Memory: about 11.8 GiB for everything but routed experts, 0.70 GiB per expert at FP4, and about + 10 GiB of extra peak during load. Under the 80 GB cap that is about 72 experts at FP4 and about + 155 at 2 bits. +- Quality at these sizes is poor, as the teacher-forced numbers predict: 48 experts give clipped + but on-topic text, and 64 fall into a loop within a 1,024-token greedy completion. Speed is + settled; quality needs the cache and 2-bit experts. +- Side effect on the machine: the server wrote `~/.omlx/bin/omlx-cluster-python`, a link to the + clone's Python. + diff --git a/scripts/v41_build/omlx_compact.py b/scripts/v41_build/omlx_compact.py new file mode 100644 index 0000000..b7ba994 --- /dev/null +++ b/scripts/v41_build/omlx_compact.py @@ -0,0 +1,105 @@ +"""A fixed resident expert set under stock oMLX, with no copy of the checkpoint (K1's speed floor). + +``make`` writes a model directory that links every file of a DeepSeek-V4.1 source checkpoint and +replaces two: ``config.json`` (``n_routed_experts`` = C) and the tensor index, which lists only +each layer's C hottest experts under the names 0..C-1. ``compact-map.json`` records which stored +tensor each of those names means and which router rows to keep. + +Any other first argument runs the oMLX CLI with three patches, all read-side: shard headers gain +the new names as aliases, and the router weight and biases are cut to the kept rows as they load. +The model then routes each token to its top experts among the C resident ones, every expert is in +memory, and oMLX's fused decode kernels apply (they do not with its own expert offload). + + python omlx_compact.py make SRC DST --experts 48 --mass reap50-pruned.npz + OMLX_COMPACT=DST/compact-map.json python omlx_compact.py serve --model-dir ... +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +EXPERT = re.compile(r"(layers\.(\d+)\.ffn\.experts)\.(\d+)\.(.+)$") +GATE = re.compile(r"layers\.(\d+)\.ffn\.gate\.(weight|bias|bias_vl)$") + + +def make(src: Path, dst: Path, experts: int, mass_file: Path) -> None: + import numpy as np + + mass = np.load(mass_file)["reap__gate_norm_sum"] # [layer, expert] routed gate mass + hot = [sorted(np.argsort(-row, kind="stable")[:experts].tolist()) for row in mass] + dst.mkdir(parents=True, exist_ok=True) + for item in src.iterdir(): + if item.name not in ("config.json", "model.safetensors.index.json") and not (dst / item.name).exists(): + (dst / item.name).symlink_to(item.resolve()) + config = json.loads((src / "config.json").read_text()) + text = config.get("text_config", config) + assert mass.shape == (text["num_hidden_layers"], text["n_routed_experts"]), mass.shape + text["n_routed_experts"] = experts + (dst / "config.json").write_text(json.dumps(config, indent=1)) + index = json.loads((src / "model.safetensors.index.json").read_text()) + weight_map, alias = {}, {} + for key, filename in index["weight_map"].items(): + m = EXPERT.match(key) + if m is None: + weight_map[key] = filename + elif int(m[3]) in hot[int(m[2])]: + new = f"{m[1]}.{hot[int(m[2])].index(int(m[3]))}.{m[4]}" + weight_map[new], alias[new] = filename, key + index["weight_map"] = weight_map + (dst / "model.safetensors.index.json").write_text(json.dumps(index)) + (dst / "compact-map.json").write_text(json.dumps({"experts": experts, "hot": hot, "alias": alias})) + print(f"{dst}: {experts} experts per layer, {len(alias)} tensors renamed") + + +def patch(spec: dict) -> None: + import mlx.core as mx + from omlx.patches.deepseek_v41 import loading, residency, storage + + alias, hot = spec["alias"], [mx.array(rows) for rows in spec["hot"]] + + def aliased(header): # a fresh dict: the stored names are read before any is overwritten + return {**header, **{new: header[old] for new, old in alias.items() if old in header}} + + opened, read_header, source_weights = storage.TensorFile.__init__, residency.header_with_offset, loading.iter_source_weights + headers = {} + + def tensor_file(self, path): + opened(self, path) + self.header = aliased(self.header) + + def header_with_offset(*args): + if args not in headers: + header, offset = read_header(*args) + headers[args] = aliased(header), offset + return headers[args] + + def iter_source_weights(*args, **kwargs): + for tensors, specs in source_weights(*args, **kwargs): + for key in tensors: + m = GATE.search(key) + if m: + tensors[key] = tensors[key][hot[int(m[1])]] + yield tensors, specs + + storage.TensorFile.__init__ = tensor_file + residency.header_with_offset = header_with_offset + loading.iter_source_weights = iter_source_weights + + +if __name__ == "__main__": + if sys.argv[1:2] == ["make"]: + p = argparse.ArgumentParser() + p.add_argument("src", type=Path) + p.add_argument("dst", type=Path) + p.add_argument("--experts", type=int, required=True) + p.add_argument("--mass", type=Path, required=True) + a = p.parse_args(sys.argv[2:]) + make(a.src.expanduser(), a.dst.expanduser(), a.experts, a.mass.expanduser()) + else: + patch(json.loads(Path(os.environ["OMLX_COMPACT"]).read_text())) + from omlx.cli import main + + sys.exit(main()) diff --git a/scripts/v41_build/omlx_counters.py b/scripts/v41_build/omlx_counters.py new file mode 100644 index 0000000..20adaf4 --- /dev/null +++ b/scripts/v41_build/omlx_counters.py @@ -0,0 +1,41 @@ +"""Run the oMLX CLI with the V4.1 expert-slot counters written to a file. + +oMLX's server logs expert-cache hits and misses per request only for its generic offload wrapper; +the V4.1 wrapper keeps the same counters but nothing reads them. This starts ``omlx`` unchanged +and appends one JSON line to ``$OMLX_COUNTERS`` whenever the totals move: + + OMLX_COUNTERS=/tmp/counters.jsonl python omlx_counters.py serve --model-dir ... +""" + +import json +import os +import sys +import threading +import time + +from omlx.cli import main +from omlx.patches.deepseek_v41 import moe_offload + +slots, made = [], moe_offload._ExpertSlots.__init__ + + +def init(self, *args, **kwargs): + made(self, *args, **kwargs) + slots.append(self) + + +def watch(path): + last = None + while True: + time.sleep(0.5) + now = [sum(getattr(s, f) for s in slots) for f in ("hits", "misses", "fetched_bytes")] + if slots and now != last: + last = now + with open(path, "a") as f: + f.write(json.dumps({"t": round(time.time(), 2), "layers": len(slots), "capacity": slots[0].plan.capacity, + "hits": now[0], "misses": now[1], "fetched_mb": round(now[2] / 1e6, 1)}) + "\n") + + +moe_offload._ExpertSlots.__init__ = init +threading.Thread(target=watch, args=(os.environ["OMLX_COUNTERS"],), daemon=True).start() +sys.exit(main()) From 95b9a8528e9fd82d91a44df5ae07646bab3fbb51 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:33:53 -0800 Subject: [PATCH 50/74] v41_build: expert cache that never waits, under stock oMLX (router as the switch, in-place slot writes) --- .../2026-11-08-v41-reap-recovery-log.md | 18 +- .../2026-11-10-v41-wow-build-plan.md | 37 +++ scripts/v41_build/omlx_compact.py | 11 +- scripts/v41_build/omlx_routed.py | 306 ++++++++++++++++++ 4 files changed, 367 insertions(+), 5 deletions(-) create mode 100644 scripts/v41_build/omlx_routed.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 554fa5d..c5db226 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -600,11 +600,11 @@ to 100; partial credit only for measured progress). | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | | 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 12 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. The rest needs the pool of 384 and the pod confirmation (E7) | | 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | -| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 0 | | +| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 5 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max at 22–26 tok/s with 156 MB of real reads per token (64 FP4 residents, pool of 192). The rest needs 2-bit experts, the pool of 384 and memory pressure | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **43** | | +| | **Total** | 100 | **48** | | ### 2026-11-09: prior art, plan, restart @@ -890,3 +890,17 @@ What it says: - Full table and notes: the build plan's K1 result. The server is stopped; nothing is running. - Rubric unchanged at 43: item 6 needs our own cache streaming on the Mac, not oMLX's. +### 2026-11-10: the no-wait cache runs + +- `scripts/v41_build/omlx_routed.py`: demand scored on the whole pool in the token's graph, + admission into spare slots by reader threads writing the stacked arrays in place, the router + switched only around finished writes. 64 FP4 residents of 192, tau 0.2, 4 spares. +- Decode 22–26 tok/s with admissions on (8.3 installs and 156 MB read per token, 75% of demanded + slots resident), 27–28 with them off, 30–31 without demand scoring; oMLX's exact offload 4.5. +- Admissions off reproduces the fixed set's 1,024-token output exactly. Slot bytes matched the + stored experts at every check (20 checks, 12,785 installs). +- Generation at 64 residents is still bad: both the fixed set and the cache loop on a prompt the + exact pool of 192 answers correctly. FP4 allows no more under the cap, so 2-bit experts in the + runtime are next (about 155 residents). +- Rubric item 6 moves from 0 to 5. Total 48. Server stopped; nothing is running. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 02bfd97..33fd2ab 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -348,3 +348,40 @@ modified). - Side effect on the machine: the server wrote `~/.omlx/bin/omlx-cluster-python`, a link to the clone's Python. +### No-wait cache, first version (2026-11-10) + +`scripts/v41_build/omlx_routed.py`, on the fixed-set base above, clone unmodified. 64 resident +experts per layer at native FP4, drawn from the pruned pool of 192; admission at gate weight 0.2, +4 spare slots per layer, least recently demanded evicted. + +How a miss costs no time: + +- Every layer also scores the whole pool's router rows on its input. The demanded experts ride in + the token's own graph, so reading them two calls later waits for nothing. +- The router is the switch. An evicted slot's bias is set so nothing routes to it; reader threads + overwrite its rows in place only after every call built before that has finished; its router + row becomes the new expert only after the bytes are in. A token never waits on a read and never + runs a half-written expert. An admitted expert serves from about three tokens after the demand. +- Stored FP4 bytes are the array's bytes, so an install is six positional reads straight into + the slot, 18.8 MB, with no copy. + +| Run | Decode tok/s | Notes | +|---|---:|---| +| fixed 64, no demand tracking | 30–31 | from K1 | +| cache code, admissions off | 26.6–28.2 | output identical to fixed 64 over 1,024 tokens; demand scoring costs 10–14% | +| cache on | 22.0–26.1 | 8.3 installs per token (156 MB), 75% of demanded slots resident | +| oMLX offload, exact, 48 slots | 4.5 | reference text below | + +- Checks: with admissions off the output equals the fixed set's, token for token. With them on, + every routed slot in the first and last layer matched its expert's stored bytes at 20 + checkpoints across 12,785 installs. A torn read in flight is excluded by the ordering rule, not + by a measurement. +- Quality is not there at this size. On a 1,024-token coding prompt the exact pool of 192 writes a + correct script; fixed 64 and cache 64 both leave the reference within a few words and end in a + loop. The cache's text is more fluent early on, which is not evidence. 64 experts is what FP4 + allows under the 80 GB cap; E1 puts the useful range at 96–144 residents. +- So the next build item is 2-bit experts in this runtime (K2, then lattice codes as slot arrays): + about 155 residents under the same cap. Also open: demand scoring through oMLX's own gate kernel + to recover the 10–14%; 23% of wanted admissions found no free slot at 4 spares; prefill runs + through the same code but was only exercised on short prompts. + diff --git a/scripts/v41_build/omlx_compact.py b/scripts/v41_build/omlx_compact.py index b7ba994..bd801b1 100644 --- a/scripts/v41_build/omlx_compact.py +++ b/scripts/v41_build/omlx_compact.py @@ -29,7 +29,9 @@ def make(src: Path, dst: Path, experts: int, mass_file: Path) -> None: import numpy as np mass = np.load(mass_file)["reap__gate_norm_sum"] # [layer, expert] routed gate mass - hot = [sorted(np.argsort(-row, kind="stable")[:experts].tolist()) for row in mass] + by_mass = [np.argsort(-row, kind="stable")[:experts].tolist() for row in mass] + hot = [sorted(row) for row in by_mass] + rank = [[by.index(e) for e in row] for by, row in zip(by_mass, hot)] # 0 is the hottest dst.mkdir(parents=True, exist_ok=True) for item in src.iterdir(): if item.name not in ("config.json", "model.safetensors.index.json") and not (dst / item.name).exists(): @@ -50,11 +52,12 @@ def make(src: Path, dst: Path, experts: int, mass_file: Path) -> None: weight_map[new], alias[new] = filename, key index["weight_map"] = weight_map (dst / "model.safetensors.index.json").write_text(json.dumps(index)) - (dst / "compact-map.json").write_text(json.dumps({"experts": experts, "hot": hot, "alias": alias})) + (dst / "compact-map.json").write_text(json.dumps( + {"experts": experts, "source": str(src.resolve()), "hot": hot, "rank": rank, "alias": alias})) print(f"{dst}: {experts} experts per layer, {len(alias)} tensors renamed") -def patch(spec: dict) -> None: +def patch(spec: dict, full: dict | None = None) -> None: import mlx.core as mx from omlx.patches.deepseek_v41 import loading, residency, storage @@ -81,6 +84,8 @@ def iter_source_weights(*args, **kwargs): for key in tensors: m = GATE.search(key) if m: + if full is not None: # the whole router, for callers that track demand + full[int(m[1]), m[2]] = tensors[key] tensors[key] = tensors[key][hot[int(m[1])]] yield tensors, specs diff --git a/scripts/v41_build/omlx_routed.py b/scripts/v41_build/omlx_routed.py new file mode 100644 index 0000000..4b07ca8 --- /dev/null +++ b/scripts/v41_build/omlx_routed.py @@ -0,0 +1,306 @@ +"""An expert cache that never waits, under stock oMLX: a miss re-routes, the fetch lands later. + +Starts from ``omlx_compact.py``'s model (C resident experts per layer as plain stacked arrays, the +router cut to their rows, so oMLX's fused decode kernels apply) and adds three things: + +- Demand. Each layer also scores the whole pool's router rows on the same input; the top experts + and their gate weights ride along in the token's graph (``mx.depends``), so reading them later + costs no wait. +- A cache policy, run once per forward call before its first layer: an expert demanded with gate + weight >= tau that is not resident is admitted into a free slot; the least recently demanded + residents are evicted to keep ``spare`` slots free. +- No-wait installs. A slot's row in the stacked weight arrays is overwritten in place by reader + threads (the stored bytes are the array's bytes). The router is the switch: an evicted slot's + bias goes to -1e30 first, its bytes are overwritten only once every forward built before that + has finished, and its router row is set to the new expert only after the bytes are in. A token + never waits for a read and never runs a half-written expert. + + OMLX_COMPACT=DIR/compact-map.json OMLX_ROUTED_LOG=log.jsonl python omlx_routed.py serve ... + +Environment: ``OMLX_ROUTED_TAU`` (default 0.2; ``inf`` keeps the starting set and must reproduce +the fixed set exactly), ``OMLX_ROUTED_SPARE`` (free slots kept per layer, default 4), +``OMLX_ROUTED_LAG`` (forward calls between a demand and its read-out, default 2: the call in +flight is never waited for), ``OMLX_ROUTED_VERIFY`` (each log line also counts routed slots whose +bytes differ from their expert's stored bytes; slow, for testing). +""" + +import json +import os +import struct +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np + +from omlx_compact import patch as compact_patch + +PROJECTIONS = ("w1", "w2", "w3") +OFF = -1e30 # a router bias no score can overcome + + +class Store: + """Positional reads of one expert's stored tensors from the source checkpoint.""" + + def __init__(self, source: Path): + self.source = source + self.index = json.loads((source / "model.safetensors.index.json").read_text())["weight_map"] + self.files: dict[str, tuple[int, dict, int]] = {} + + def slab(self, key: str) -> tuple[int, int, int]: + name = self.index[key] + if name not in self.files: + fd = os.open(self.source / name, os.O_RDONLY) + size = struct.unpack(" int: + return self.pread_into(*self.slab(key), view) + + @staticmethod + def pread_into(fd: int, offset: int, nbytes: int, view: np.ndarray) -> int: + """Fill ``view`` from the file. Positional, so reader threads may call it.""" + target = memoryview(view.reshape(-1).view(np.uint8)) + assert target.nbytes == nbytes, (target.nbytes, nbytes) + done = 0 + while done < nbytes: + count = os.preadv(fd, [target[done:]], offset + done) + assert count > 0, "truncated tensor" + done += count + return nbytes + + +class Layer: + def __init__(self, index, moe, full_weight, full_bias, hot, rank, mx): + self.index, self.gate, self.experts = index, moe.gate, moe.experts + self.full_weight = full_weight.astype(mx.float32) # [pool, dim] + self.full_bias = full_bias.astype(mx.float32) + self.typed_weight, self.typed_bias = full_weight, full_bias # the rows a slot's router entry is copied from + mx.eval(self.full_weight, self.full_bias) + self.slot_expert = np.array(hot, dtype=np.int64) # -1: unrouted + self.slot_of = {e: s for s, e in enumerate(hot)} + self.last_use = -1.0 - np.array(rank, dtype=np.float64) # colder starting experts go first + self.cooling: list[tuple[int, int]] = [] # (slot, forward call that unrouted it) + self.free: list[int] = [] + self.reading: dict[int, tuple[int, list]] = {} # expert -> (slot, futures) + self.views = {} + for proj in PROJECTIONS: + lin = getattr(self.experts, proj) + assert lin["weight"].dtype == mx.uint32 and lin["scales"].dtype == mx.uint8, proj + self.views[proj] = (lin, lin["weight"], lin["scales"], + np.asarray(memoryview(lin["weight"])), np.asarray(memoryview(lin["scales"]))) + + def views_current(self) -> bool: + """The host views still alias the arrays the kernels read.""" + for lin, weight, scales, weight_view, scales_view in self.views.values(): + for owner, name, view in ((weight, "weight", weight_view), (scales, "scales", scales_view)): + if lin[name] is not owner or np.frombuffer(memoryview(owner), np.uint8).ctypes.data != view.ctypes.data: + return False + return True + + +class Cache: + def __init__(self, model, config, spec, full, mx): + self.mx, self.config = mx, config + self.tau = float(os.environ.get("OMLX_ROUTED_TAU", "0.2")) + self.spare = int(os.environ.get("OMLX_ROUTED_SPARE", "4")) + self.lag = int(os.environ.get("OMLX_ROUTED_LAG", "2")) + self.log_path = os.environ.get("OMLX_ROUTED_LOG") + assert config.score_func == "sqrtsoftplus" and self.lag >= 1 + self.store = Store(Path(spec["source"])) + self.pool = ThreadPoolExecutor(max_workers=8, thread_name_prefix="routed-io") + blocks = model.language_model.layers + self.layers = [Layer(i, blk.ffn, full[i, "weight"], full[i, "bias"], spec["hot"][i], spec["rank"][i], mx) + for i, blk in enumerate(blocks)] + self.by_gate = {id(layer.gate): layer for layer in self.layers} + self.step = 0 + self.pending: dict[int, list] = {} # forward call -> per-layer (index, weight) demand arrays + self.stats = dict.fromkeys(("calls", "tokens", "demand", "resident", "wanted", "admitted", "installed", + "evicted", "bytes", "no_slot"), 0) + self.started = time.time() + for key in {name: key for key, name in self.store.index.items() if ".ffn.experts." in key}.values(): + self.store.slab(key) # open every expert shard now, not during a token + self.check_layout() + + def check_layout(self): + """A resident expert's rows must equal its stored bytes: installs rely on it.""" + for layer in self.layers[:: max(1, len(self.layers) // 4)]: + expert = int(layer.slot_expert[0]) + for proj in PROJECTIONS: + for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): + key = f"layers.{layer.index}.ffn.experts.{expert}.{proj}.{field}" + stored = np.empty(view[0].nbytes, np.uint8) + self.store.read_into(key, stored) + assert np.array_equal(stored, view[0].reshape(-1).view(np.uint8)), f"layout differs: {key}" + + # --- demand, recorded while a forward call is built ------------------------------------- + + def demand(self, layer, x): + mx, c = self.mx, self.config + raw = (x.reshape(-1, x.shape[-1]).astype(mx.float32) @ layer.full_weight.T) / c.gate_temp + scores = mx.sqrt(mx.logaddexp(raw, 0)) + index = mx.argpartition(-(scores + layer.full_bias), kth=c.n_activated_experts - 1, axis=-1) + index = index[..., : c.n_activated_experts] + return index, mx.take_along_axis(scores, index, -1) + + def observe(self, gate, x, tied): + """Record this layer's demand and tie it to ``tied`` so one eval covers both.""" + layer = self.by_gate[id(gate)] + index, weight = self.demand(layer, x) + self.pending.setdefault(self.step, []).append((index, weight)) + tied = self.mx.depends(tied, [index, weight]) + return tied[0] if isinstance(tied, (list, tuple)) else tied + + # --- policy, run before the first layer of each forward call ---------------------------- + + def begin(self): + self.step += 1 + ready = self.step - self.lag + if self.tau != float("inf") and ready in self.pending: + self.harvest(ready, self.pending.pop(ready)) + for old in [s for s in self.pending if s < ready]: + del self.pending[old] + if self.log_path and self.step % 64 == 0: + self.write_log() + + def harvest(self, call, demand): + c = self.config + self.stats["calls"] += 1 + # Reading this call's demand means every earlier call has left the GPU. + for layer, (index, weight) in zip(self.layers, demand): + index, weight = np.asarray(index), np.asarray(weight, dtype=np.float64) + weight = weight / (weight.sum(-1, keepdims=True) + 1e-20) * c.route_scale + if layer.index == 0: + self.stats["tokens"] += index.shape[0] + self.stats["demand"] += index.size + strong = weight >= self.tau + wanted: dict[int, float] = {} + for expert, w in zip(index[strong].tolist(), weight[strong].tolist()): + wanted[expert] = max(w, wanted.get(expert, 0.0)) + self.stats["resident"] += sum(e in layer.slot_of for e in index.reshape(-1).tolist()) + self.install(layer) + missing = [] + for expert, w in wanted.items(): + slot = layer.slot_of.get(expert) + if slot is not None: + layer.last_use[slot] = call + elif expert not in layer.reading: + missing.append((w, expert)) + self.stats["wanted"] += len(missing) + layer.free += [slot for slot, since in layer.cooling if since <= call] + layer.cooling = [(slot, since) for slot, since in layer.cooling if since > call] + if missing and layer.free and not layer.views_current(): + raise RuntimeError(f"layer {layer.index}: expert arrays moved; cannot install in place") + for _, expert in sorted(missing, reverse=True): + if not layer.free: + self.stats["no_slot"] += 1 + continue + slot = layer.free.pop() + futures = [] + for proj in PROJECTIONS: + base = f"layers.{layer.index}.ffn.experts.{expert}.{proj}." + for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): + futures.append(self.pool.submit(Store.pread_into, *self.store.slab(base + field), view[slot])) + layer.reading[expert] = (slot, futures) + self.stats["admitted"] += 1 + self.evict(layer, set(wanted)) + + def install(self, layer) -> None: + """Route the experts whose bytes have landed. The router arrays are replaced, never + written in place and never evaluated here: calls already built keep the old ones, and + the next call's own eval computes the new ones without a wait.""" + done = [e for e, (_, futures) in layer.reading.items() if all(f.done() for f in futures)] + for expert in done: + slot, futures = layer.reading.pop(expert) + self.stats["bytes"] += sum(f.result() for f in futures) + layer.gate.weight[slot] = layer.typed_weight[expert] + layer.gate.bias[slot] = layer.typed_bias[expert] + layer.slot_expert[slot], layer.slot_of[expert] = expert, slot + layer.last_use[slot] = self.step + self.stats["installed"] += 1 + + def evict(self, layer, protected) -> None: + """Unroute the least recently demanded residents until ``spare`` slots are free or coming.""" + short = self.spare - len(layer.free) - len(layer.cooling) + if short <= 0: + return + order = np.argsort(layer.last_use, kind="stable") + victims = [int(s) for s in order if layer.slot_expert[s] >= 0 and int(layer.slot_expert[s]) not in protected] + for slot in victims[:short]: + del layer.slot_of[int(layer.slot_expert[slot])] + layer.slot_expert[slot] = -1 + layer.last_use[slot] = np.inf + layer.gate.bias[slot] = OFF + # Calls built before this one may still route here; its bytes may change only once + # the call being built now is known to have finished. + layer.cooling.append((slot, self.step)) + self.stats["evicted"] += 1 + + def mismatched(self) -> int: + """Routed slots (first and last layer) whose rows differ from their expert's stored bytes.""" + bad = 0 + for layer in (self.layers[0], self.layers[-1]): + for slot, expert in enumerate(layer.slot_expert.tolist()): + for proj in PROJECTIONS if expert >= 0 else (): + for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): + stored = np.empty(view[slot].nbytes, np.uint8) + self.store.read_into(f"layers.{layer.index}.ffn.experts.{expert}.{proj}.{field}", stored) + bad += not np.array_equal(stored, view[slot].reshape(-1).view(np.uint8)) + return bad + + def write_log(self): + row = {"t": round(time.time() - self.started, 2), "step": self.step, **self.stats, + "reading": sum(len(layer.reading) for layer in self.layers)} + if os.environ.get("OMLX_ROUTED_VERIFY"): + row["mismatched"] = self.mismatched() + with open(self.log_path, "a") as f: + f.write(json.dumps(row) + "\n") + + +def patch(spec: dict) -> None: + import mlx.core as mx + from omlx.patches.deepseek_v41 import decode_moe, language, loading + + full: dict = {} + compact_patch(spec, full) + state: dict = {} + load, fused_gate, gate_call = loading.load, decode_moe._gate, language.Gate.__call__ + + def load_routed(*args, **kwargs): + model, processor = load(*args, **kwargs) + state["cache"] = Cache(model, model.language_model.layers[0].ffn.gate._config, spec, full, mx) + full.clear() + return model, processor + + def gate_fused(moe, x, config): # one-token and batched fused decode + cache = state.get("cache") + if cache is None or id(moe.gate) not in cache.by_gate: + return fused_gate(moe, x, config) + if cache.by_gate[id(moe.gate)].index == 0: + cache.begin() + scores, quantized = fused_gate(moe, x, config) + return cache.observe(moe.gate, x, scores), quantized + + def gate_plain(self, x, image_mask): # prefill and every unfused path + cache = state.get("cache") + if cache is None or id(self) not in cache.by_gate: + return gate_call(self, x, image_mask) + if cache.by_gate[id(self)].index == 0: + cache.begin() + index, weights = gate_call(self, x, image_mask) + return cache.observe(self, x, index), weights + + loading.load = load_routed + decode_moe._gate = gate_fused + language.Gate.__call__ = gate_plain + + +if __name__ == "__main__": + patch(json.loads(Path(os.environ["OMLX_COMPACT"]).read_text())) + from omlx.cli import main + + sys.exit(main()) From 21603b5936dd25ff76663cfaf4a44bbc06c48d76 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:39:07 -0800 Subject: [PATCH 51/74] K2: lattice kernel matches oMLX's expert formats per expert at V4.1 shape; benchmark script --- .../2026-11-08-v41-reap-recovery-log.md | 8 + .../2026-11-10-v41-wow-build-plan.md | 30 +++- scripts/v41_build/bench_expert_formats.py | 143 ++++++++++++++++++ 3 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 scripts/v41_build/bench_expert_formats.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index c5db226..8ed63f8 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -904,3 +904,11 @@ What it says: runtime are next (about 155 residents). - Rubric item 6 moves from 0 to 5. Total 48. Server stopped; nothing is running. +### 2026-11-10: K2, the lattice kernel against oMLX's formats + +- At V4.1 shape on the M4 Max, per routed expert: E8P lattice 2-bit 0.037 ms, oMLX affine 2-bit + 0.047, oMLX MXFP4 0.042, oMLX affine 3-bit 0.036 (three runs, within 5%). K2 passes. +- The lattice expert is 9.0 MB against 11.1 for affine 2-bit: about 155 residents under the cap + against about 127. Table and limits in the build plan's K2 result. +- Rubric unchanged at 48. Nothing is running. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 33fd2ab..47c0b00 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -136,7 +136,7 @@ Red-team of the chosen plan: | # | Test | Kills | |---|---|---| | K1 | Stock oMLX on this Mac, local pruned checkpoint: tok/s, hit rate, resident memory, and its no-miss floor. **Passed 2026-11-10:** the loader takes the 192-expert checkpoint; with oMLX's own offload at 48 of 192 experts it decodes at 3.3–4.5 tok/s (hit rate 78%, about 0.9 GB read per token) and is not on the fused path; with a fixed 48 or 64 experts and a sliced router it decodes at 30–32 tok/s in the same memory. | The base, if the no-miss floor is under about 10 tok/s: switch to Rapid-MLX's decode kernels. | -| K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. | Our lattice kernel in the runtime: ship affine 2-bit instead. | +| K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. **Passed 2026-11-10:** per routed expert the lattice kernel takes 0.037 ms, oMLX's affine 2-bit 0.047 and its FP4 0.042; the lattice expert is 9.0 MB against 11.1. | Our lattice kernel in the runtime: ship affine 2-bit instead. | | K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod. **Done 2026-11-10:** KL 0.323 at 250 and 0.235 at 288 (0.532 at 192, 1.123 at 96), native precision, no cache. | Nothing; it sets how much the cache has to recover. At the 80 GB cap (about 160 experts) a fixed set costs roughly 0.6–0.7 by interpolation, so the cap matters more than the code density. | | K4 | E1: expert cache with re-route against static sets, teacher-forced. **Passed 2026-11-10:** the cache beats the static set by 0.056 KL at 144 residents and 0.231 at 96 (pool of 192, 2 bits); a cache of 96 equals a static 144; the oracle gap (0.020) is all in the first 256 tokens. | The quality case for the cache, if it is not better than a static set. | | K5 | 20 lines: an MLX array over a 1 GB memory-mapped file through the E8P kernel; check footprint and `mincore`. | Idea 3 (page cache), if it copies. | @@ -385,3 +385,31 @@ How a miss costs no time: to recover the 10–14%; 23% of wanted admissions found no free slot at 4 spares; prefill runs through the same code but was only exercised on short prompts. +### K2 result (2026-11-10) + +`scripts/v41_build/bench_expert_formats.py`, M4 Max, one process, MLX 0.32.4, 64 experts, 40 +chained MoEs per eval, three runs (medians agree within 5%). oMLX rows time its fused decode MoE +(router, routed experts, shared expert) on random weights; the lattice row times this repo's +`e8p_m1` kernel on layer 20's real 2-bit codes, routed experts only. Cost per expert is the +difference between top-6 and top-1, over five. + +| Routed-expert format | Bits per weight | Expert, MB | MoE at top-6, ms | Per expert, ms | Fixed, ms | +|---|---:|---:|---:|---:|---:| +| oMLX MXFP4 (checkpoint format) | 4.25 | 18.8 | 0.39–0.41 | 0.042–0.045 | 0.145 | +| oMLX affine 2-bit, group 64 | 2.5 | 11.1 | 0.41 | 0.046–0.048 | 0.124–0.134 | +| oMLX affine 3-bit, group 64 | 3.5 | 15.5 | 0.35–0.36 | 0.035–0.036 | 0.140–0.145 | +| E8P lattice 2-bit (routed only) | 2.04 | 9.0 | 0.29 | 0.036–0.038 | 0.065–0.069 | + +- The lattice kernel is not slower per expert than any oMLX format, so K2 does not kill it. It + is 3.3 times this repo's generic kernel at this shape (0.95 ms). +- What it buys is residents, not speed: under the 80 GB cap about 155 lattice experts fit + against about 127 at affine 2-bit. E1 puts a cache of 144 within 0.009 KL of the whole pool of + 192, so the lattice format clears that line and affine 2-bit does not. +- Not measured: the lattice kernel inside oMLX's decode step. The router and shared expert then + run as their own dispatches; from the fixed costs above the MoE would take about 0.43 ms + against 0.39–0.41, roughly 3–5% of a token. +- Side note: oMLX's 3-bit affine MoE is its fastest, ahead of FP4. +- Next build item: lattice experts as slot arrays in the cache. That needs the codes stored as + raw per-expert bytes (they are `.npz` today), the routed half of the fused MoE replaced by + `e8p_m1`, and codes for the experts outside the pruned pool. + diff --git a/scripts/v41_build/bench_expert_formats.py b/scripts/v41_build/bench_expert_formats.py new file mode 100644 index 0000000..2bcf05f --- /dev/null +++ b/scripts/v41_build/bench_expert_formats.py @@ -0,0 +1,143 @@ +"""K2: one decode-step MoE at V4.1 shape, by routed-expert format, in one process. + +- oMLX's fused decode kernels (``decode_moe.fused_decode_moe``: router, top-k routed experts and + the shared expert in three dispatches) with routed experts as MXFP4 (the checkpoint's own + format) and as oMLX's affine codes at 2 and 3 bits, group 64. Random weights. +- This repo's E8P lattice kernel (``ramp.kernels.e8p_m1``; routed experts only, no router and + no shared expert) on real 2-bit codes of one layer. + +Each sample chains ``--chain`` dependent MoEs before one ``mx.eval``, as a decode step does. +Running each at top-k 6 and top-k 1 separates the cost per routed expert from the fixed cost. + + PYTHONPATH=src:benchmarks ~/Development/_refs/omlx/.venv/bin/python \ + scripts/v41_build/bench_expert_formats.py --config ~/Models/v41-reap50/config.json \ + --e8p ~/Models/v41-reap-run/quant/e8p-podimp/layer-20.npz +""" + +import argparse +import json +import statistics +import time +from dataclasses import replace +from pathlib import Path + +import mlx.core as mx +import numpy as np + +EXPERTS = 64 + + +def median_ms(step, chain, iterations=30, warmup=10): + def sample(): + started = time.perf_counter() + step(chain) + return (time.perf_counter() - started) * 1e3 / chain + + for _ in range(warmup): + sample() + return statistics.median(sample() for _ in range(iterations)) + + +def omlx_moe(config, mode, bits, group, rng): + from omlx.patches.deepseek_v41.language import MoE + from omlx.patches.deepseek_v41.quantization import QuantizedProjection + + def projection(out, inp, lead, mode, bits, group): + weight = mx.array(rng.integers(0, 1 << 32, size=(*lead, out, inp * bits // 32), dtype=np.uint32)) + if mode == "affine": + shape = (*lead, out, inp // group) + return QuantizedProjection(weight, mx.full(shape, 0.01, mx.bfloat16), bits, mode, + biases=mx.full(shape, -0.005 * (2**bits - 1), mx.bfloat16), group_size=group) + return QuantizedProjection(weight, mx.full((*lead, out, inp // 32), 118, mx.uint8), bits, mode) # 2^-9 + + moe = MoE(config) + dim, inter = config.dim, config.moe_inter_dim + for name, out, inp in (("w1", inter, dim), ("w3", inter, dim), ("w2", dim, inter)): + setattr(moe.experts, name, projection(out, inp, (EXPERTS,), mode, bits, group)) + setattr(moe.shared_experts, name, projection(out, inp, (), "mxfp8", 8, 32)) + moe.gate.weight = mx.array(rng.normal(size=(EXPERTS, dim)).astype(np.float32)).astype(mx.bfloat16) + moe.gate.bias = mx.zeros((EXPERTS,)) + arrays = [moe.gate.weight, moe.gate.bias] + nbytes = 0 + for name in ("w1", "w3", "w2"): + p, s = getattr(moe.experts, name), getattr(moe.shared_experts, name) + fields = [p.weight, p.scales] + ([p.biases] if "biases" in p else []) + arrays += fields + [s.weight, s.scales] + nbytes += sum(a.nbytes for a in fields) // EXPERTS + mx.eval(arrays) + return moe, nbytes + + +def bench_omlx(config, mode, bits, group, top_k, chain, seed=0): + from omlx.patches.deepseek_v41 import decode_moe + + rng = np.random.default_rng(seed) + c = replace(config, n_routed_experts=EXPERTS, n_activated_experts=top_k) + moe, expert_bytes = omlx_moe(c, mode, bits, group, rng) + x0 = mx.array(rng.normal(size=(1, 1, c.dim)).astype(np.float32)).astype(mx.bfloat16) + assert decode_moe.supported(moe, x0, c), (mode, bits, group) + state = {"x": x0} + + def step(n): + x = state["x"] + for _ in range(n): + out = decode_moe.fused_decode_moe(moe, x, c)[0].astype(mx.float32) + x = (out / (mx.max(mx.abs(out)) + 1e-6) * 3).astype(mx.bfloat16) # a new input, so new routes + mx.eval(x) + state["x"] = x + + ms = median_ms(step, chain) + assert bool(mx.all(mx.isfinite(state["x"].astype(mx.float32)))), "non-finite activations" + return ms, expert_bytes + + +def bench_e8p(path, top_k, chain, sweep=((4, 1), (8, 1), (4, 4), (2, 4))): + from bench_e8p_m1 import PROJECTIONS, projection_bytes_per_expert, run + from keep.vq.e8 import e8p_packed_abs_grid + from ramp.nn.switch_linear import QuantizedVQSwitchLinear + + z, layers = np.load(path), {} + for name, key in zip(PROJECTIONS, ("w1", "w3", "w2")): + codes, scales = z[f"{key}_codes"][:EXPERTS], z[f"{key}_scales"][:EXPERTS] + layers[name] = QuantizedVQSwitchLinear( + input_dims=codes.shape[2] * 8, output_dims=codes.shape[1], num_experts=EXPERTS, + codes=mx.array(codes), scales=mx.array(scales), codebook=mx.array(e8p_packed_abs_grid()), + group_size=codes.shape[2] * 8 // scales.shape[2], code_bits=16) + mx.eval([layer.parameters() for layer in layers.values()]) + result = run(layers, top_k=top_k, iterations=30, warmup=10, seed=0, sweep=sweep, ffns_per_sync=chain) + assert result["outputs_agree"], result["configs"] + best = result["best"] + return (best["median_ms_per_ffn"], sum(projection_bytes_per_expert(layers[p]) for p in PROJECTIONS), + (best["rows_per_simdgroup"], best["simdgroups"]), result["production_median_ms_per_ffn"]) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--config", type=Path, required=True) + p.add_argument("--e8p", type=Path, required=True) + p.add_argument("--chain", type=int, default=40) + p.add_argument("--out", type=Path) + args = p.parse_args() + from omlx.patches.deepseek_v41.config import ModelConfig + + config = ModelConfig.from_dict(json.loads(args.config.expanduser().read_text())) + rows = [] + for label, mode, bits, group in (("omlx mxfp4", "mxfp4", 4, 32), ("omlx affine 2-bit g64", "affine", 2, 64), + ("omlx affine 3-bit g64", "affine", 3, 64)): + (ms6, nbytes), (ms1, _) = (bench_omlx(config, mode, bits, group, k, args.chain) for k in (6, 1)) + rows.append({"format": label, "ms_top6": ms6, "ms_top1": ms1, "expert_mb": nbytes / 1e6}) + mx.clear_cache() + (ms6, nbytes, best, production), (ms1, *_) = (bench_e8p(args.e8p.expanduser(), k, args.chain) for k in (6, 1)) + rows.append({"format": "e8p lattice 2-bit (routed experts only)", "ms_top6": ms6, "ms_top1": ms1, + "expert_mb": nbytes / 1e6, "rows_per_simdgroup,simdgroups": best, "generic_kernel_ms_top6": production}) + for row in rows: + row["ms_per_expert"] = (row["ms_top6"] - row["ms_top1"]) / 5 + row["ms_fixed"] = row["ms_top1"] - row["ms_per_expert"] + row["expert_gb_per_s"] = row["expert_mb"] / 1e3 / (row["ms_per_expert"] / 1e3) + print(json.dumps({k: round(v, 4) if isinstance(v, float) else v for k, v in row.items()})) + if args.out: + args.out.write_text(json.dumps({"mlx": mx.__version__, "chain": args.chain, "experts": EXPERTS, "rows": rows}, indent=1)) + + +if __name__ == "__main__": + main() From 8796962b6e19ecba4dea9c4e2d81dd08cb15ce72 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:49:58 -0800 Subject: [PATCH 52/74] v41_build: 2-bit lattice experts in the no-wait cache under stock oMLX (144 residents, 26-28 tok/s on the M4 Max) --- .../2026-11-08-v41-reap-recovery-log.md | 19 +- .../2026-11-10-v41-wow-build-plan.md | 66 +++ scripts/v41_build/omlx_compact.py | 6 +- scripts/v41_build/omlx_lattice.py | 420 ++++++++++++++++++ scripts/v41_build/omlx_routed.py | 112 +++-- 5 files changed, 575 insertions(+), 48 deletions(-) create mode 100644 scripts/v41_build/omlx_lattice.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 8ed63f8..c6abdb8 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -600,11 +600,11 @@ to 100; partial credit only for measured progress). | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | | 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 12 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. The rest needs the pool of 384 and the pod confirmation (E7) | | 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | -| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 5 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max at 22–26 tok/s with 156 MB of real reads per token (64 FP4 residents, pool of 192). The rest needs 2-bit experts, the pool of 384 and memory pressure | +| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 7 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max with 144 2-bit lattice residents of the pool of 192 at 26–28 tok/s, 20 MB of real reads per token, 62 GiB (FP4, 64 residents: 22–26 tok/s). The rest needs the pool of 384 and memory pressure | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **48** | | +| | **Total** | 100 | **50** | | ### 2026-11-09: prior art, plan, restart @@ -912,3 +912,18 @@ What it says: against about 127. Table and limits in the build plan's K2 result. - Rubric unchanged at 48. Nothing is running. +### 2026-11-10: 2-bit lattice experts in the no-wait cache + +- `scripts/v41_build/omlx_lattice.py`: 144 lattice residents per layer of the pool of 192 under + stock oMLX, codes read straight from the quantizer's `.npz` files, the router scoring the + whole pool with residency as a bias mask. Self-check against host-decoded matrices within + 0.3%; installs byte-exact (17 checks, 2,431 installs on the real model). +- Decode 26.0–27.9 tok/s with the cache on (2.2 installs and 20 MB read per token, 96% of + demanded slots resident), 28.6–30.1 with the policy off. 62 GiB steady, 68–69 peak. A + 4,341-token prompt prefills at 111–192 tok/s. +- With thinking off the model writes a duplicate-file script that runs correctly and summarizes + a 4,341-token source file accurately. With thinking on it does not reach an answer in 4,096 + tokens, with the cache on or off, greedy or sampled; the cache is not the cause, and 2-bit + codes are not yet separated from pruning. +- Rubric item 6 moves from 5 to 7. Total 50. Server stopped; nothing is running. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 47c0b00..e721a05 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -413,3 +413,69 @@ difference between top-6 and top-1, over five. raw per-expert bytes (they are `.npz` today), the routed half of the fused MoE replaced by `e8p_m1`, and codes for the experts outside the pruned pool. +### Lattice experts in the cache (2026-11-10) + +`scripts/v41_build/omlx_lattice.py`. oMLX (pinned clone, unmodified) loads a six-expert model +made by `omlx_compact.py make --experts 6`; after the load each layer's routed experts are +replaced by 144 lattice experts held as stacked code and scale arrays. + +- Storage needed no conversion. The quantizer's `layer-NN.npz` files are stored uncompressed, so + an expert's codes are a fixed byte range of the file and an install is a positional read into + the slot's rows. +- The router keeps every row of the pool and scores all 192 once per token with oMLX's gate + kernel. One small kernel takes two top-6's of those scores: among the residents (every other + expert's bias is -1e30), which is what runs, and over the pool, which is the demand the policy + reads two calls later. An install or eviction changes one bias and one entry of an + expert-to-slot table. This replaced the separate demand matmul of the FP4 cache. +- Decode step: gate kernel, top-k kernel, `e8p_m1` for w1 and w3, oMLX's SwiGLU and act-quant + kernel, `e8p_m1` for w2, then oMLX's shared-expert kernels, which add the routed sum. The + shared up kernel runs with no routed column (`TOPK` 0). +- Prefill: oMLX's unfused MoE with the routed experts on this repo's sorted-route kernels. +- `omlx_routed.py` is now the policy with the FP4 storage as one subclass; the lattice cache is + the other. + +Self-check (`omlx_lattice.py check`, two layers of real codes, random routers and shared +experts): the decode step is within 0.3% of the same MoE computed from matrices decoded on the +host, the prefill path within 0.1% of the decode path, and after 83 installs no slot differs +from its stored bytes. The decode-step MoE takes 0.40–0.42 ms against 0.38–0.41 for oMLX's +fused FP4 MoE in the same harness. The lattice kernel is 1.7 times slower on fp32 activation +rows than on 16-bit ones (0.51 against 0.29 ms for the routed half), so it gets bf16. + +Serving, M4 Max, pool of 192, 144 residents per layer, tau 0.2, 4 spares: + +| | Decode, tok/s | | +|---|---:|---| +| Policy off (fixed 144) | 28.6–30.1 | text reproducible run to run | +| Cache on | 26.0–27.9 | 2.2 installs and 20 MB read per token; 96% of demanded slots resident | +| oMLX FP4, fixed 64 (K1) | 30–31 | | +| FP4 cache, 64 residents | 22–26 | 8.3 installs and 156 MB per token; 75% resident | + +- Memory: 62 GiB steady, 68–69 GiB peak during the load, which takes about 25 s. +- Prefill: a 4,341-token prompt took 22.6 s and 39.1 s (192 and 111 tok/s, two runs, no KV + reuse). Prompts of 38–64 tokens reach the first token in 1.4–2.3 s. A prefill is one forward + call, so the cache cannot adapt inside it; over the runs with the long prompt 83% of demanded + slots were resident. +- Policy cost: 1.5 ms of Python per token, and about 2 ms more that is not attributed yet. + Reading the demand through a lazy reshape forced a GPU sync per layer and cost another 10% + (22.6–25.8 tok/s) until the raw kernel outputs were kept instead. +- Installs on the real model: 17 checks over 2,431 installs, no slot differing from its bytes. +- 5–11% of wanted admissions found no free slot at 4 spares. + +Text, same prompt as before (a script that finds duplicate files by SHA-256): + +- Thinking off, temperature 0: a complete script in 1,035–1,182 tokens, and it runs correctly + on a test tree. A 4,341-token prompt holding `omlx_routed.py` got an accurate five-bullet + summary and a plausible bug risk. +- Thinking on: no answer within 4,096 tokens in any of three runs. Greedy with the cache on is + coherent for about 1,000 tokens and then repeats itself; default sampling does not repeat but + keeps reviewing code it drafted in its reasoning; greedy with the policy off (fixed 144) + also runs out. So the cache is not the cause. The exact pool of 192 at FP4 starts writing + after about 500 characters of reasoning on this prompt (K1's 1,024-token run); it was not + run to 4,096 tokens, so 2-bit codes and pruning are not separated here. +- With the cache on, greedy output is not reproducible: install timing changes routing. Two + runs diverged after 18 characters; a third matched the second. + +Not done: the pool of 384 (codes exist only for the pruned 192), more than 144 residents (160 +would be about 67 GiB steady), memory pressure, and multi-token verify windows in serving (the +self-check covers 8 rows). + diff --git a/scripts/v41_build/omlx_compact.py b/scripts/v41_build/omlx_compact.py index bd801b1..aa50072 100644 --- a/scripts/v41_build/omlx_compact.py +++ b/scripts/v41_build/omlx_compact.py @@ -29,7 +29,8 @@ def make(src: Path, dst: Path, experts: int, mass_file: Path) -> None: import numpy as np mass = np.load(mass_file)["reap__gate_norm_sum"] # [layer, expert] routed gate mass - by_mass = [np.argsort(-row, kind="stable")[:experts].tolist() for row in mass] + order = [np.argsort(-row, kind="stable").tolist() for row in mass] # every expert, hottest first + by_mass = [row[:experts] for row in order] hot = [sorted(row) for row in by_mass] rank = [[by.index(e) for e in row] for by, row in zip(by_mass, hot)] # 0 is the hottest dst.mkdir(parents=True, exist_ok=True) @@ -53,7 +54,8 @@ def make(src: Path, dst: Path, experts: int, mass_file: Path) -> None: index["weight_map"] = weight_map (dst / "model.safetensors.index.json").write_text(json.dumps(index)) (dst / "compact-map.json").write_text(json.dumps( - {"experts": experts, "source": str(src.resolve()), "hot": hot, "rank": rank, "alias": alias})) + {"experts": experts, "source": str(src.resolve()), "hot": hot, "rank": rank, "order": order, + "alias": alias})) print(f"{dst}: {experts} experts per layer, {len(alias)} tensors renamed") diff --git a/scripts/v41_build/omlx_lattice.py b/scripts/v41_build/omlx_lattice.py new file mode 100644 index 0000000..2e7cb6c --- /dev/null +++ b/scripts/v41_build/omlx_lattice.py @@ -0,0 +1,420 @@ +"""The no-wait expert cache (``omlx_routed.py``) with 2-bit lattice experts, under stock oMLX. + +The routed experts are this repo's E8P codes (2.04 bits per weight, 9.0 MB an expert against +18.8 MB at MXFP4), so about twice as many stay resident. oMLX loads a small model +(``omlx_compact.py make --experts 6``); after the load each layer's routed experts are replaced: + +- Storage. A slot is one row of six stacked arrays (codes and scales of w1, w3, w2). The bytes + come straight out of the quantizer's ``layer-NN.npz`` files, which are stored uncompressed, so + an install is a positional read into the slot's rows, as in the MXFP4 cache. +- Router. It keeps every row of the pool and scores all of it once per token (oMLX's gate + kernel). One small kernel then takes two top-k's of those scores: among the resident experts + (the bias of every other expert is -1e30), which is what runs, and in the whole pool, which + is the demand the cache policy reads later. Installing or evicting an expert changes its bias + and its entry in the expert-to-slot table; the router rows never move. +- Decode step. Router, then this repo's ``e8p_m1`` kernels for the routed experts, then oMLX's + own kernels for the shared expert, which also add the routed sum. +- Prefill. oMLX's unfused MoE, with the routed experts on this repo's sorted-route kernels. + + OMLX_COMPACT=DIR/compact-map.json OMLX_LATTICE=CODES_DIR python omlx_lattice.py serve ... + python omlx_lattice.py check --config CONFIG.json --codes CODES_DIR + +Environment: ``OMLX_LATTICE_SLOTS`` (resident experts per layer, default 144) and the policy +settings of ``omlx_routed.py``. ``check`` compares one layer's decode step with dense matrices +decoded on the host, the prefill path with the decode path, and slots after installs with their +stored bytes, then times the decode-step MoE. +""" + +import json +import os +import struct +import sys +import time +import zipfile +from dataclasses import replace +from functools import cache +from pathlib import Path +from types import SimpleNamespace + +import mlx.core as mx +import numpy as np +from mlx import nn + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + +from omlx.patches.deepseek_v41 import decode_moe as DM +from omlx.patches.deepseek_v41 import language, loading +from omlx.patches.deepseek_v41.activation import quantize_swiglu_activation +from omlx.patches.deepseek_v41.quantization import quantize_activation +from omlx_compact import patch as compact_patch +from omlx_routed import OFF, Cache, Store + +from keep.vq.e8 import decode_weight_matrix, e8p_packed_abs_grid +from ramp.kernels.e8p_m1 import e8p_m1_per_route, e8p_m1_token_layout +from ramp.nn.switch_linear import QuantizedVQSwitchLinear + +TENSORS = [(proj, field) for proj in ("w1", "w3", "w2") for field in ("codes", "scales")] + +# One threadgroup per token. Top-k among the residents (slots and routing weights, as oMLX's +# up kernel computes them) and top-k in the whole pool (expert numbers and raw scores). +_TOPK = r""" + threadgroup int chosen[8]; + const uint tid = thread_index_in_threadgroup, b = threadgroup_position_in_grid.x; + const device float* sc = scores + b * NE; + topk_select(sc, bias, chosen, tid); + if (tid == 0) { + float wsum = 0.0f; + for (int j = 0; j < TOPK; ++j) wsum += sc[chosen[j]]; + for (int j = 0; j < TOPK; ++j) { + slot[b * TOPK + j] = slot_of[chosen[j]]; + weight[b * TOPK + j] = sc[chosen[j]] / (wsum + 1e-20f) * route_scale[0]; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + topk_select(sc, pool_bias, chosen, tid); + if (tid == 0) + for (int j = 0; j < TOPK; ++j) { + want[b * TOPK + j] = uint(chosen[j]); + want_score[b * TOPK + j] = sc[chosen[j]]; + } +""" + + +@cache +def _topk_kernel(): + return mx.fast.metal_kernel( + name="v41_lattice_topk", input_names=["scores", "bias", "pool_bias", "slot_of", "route_scale"], + output_names=["slot", "weight", "want", "want_score"], source=_TOPK, header=DM._HEADER) + + +@cache +def _unused(dtype): + """Stands in for the routed MXFP4 inputs of oMLX's up kernel, which runs no routed column.""" + a = mx.zeros((8,), dtype) + mx.eval(a) + return a + + +class Codes: + """Where each expert's codes and scales sit inside the ``layer-NN.npz`` files.""" + + def __init__(self, root: Path, layers: int): + self.files = [] + for i in range(layers): + path = root / f"layer-{i:02d}.npz" + fd, members = os.open(path, os.O_RDONLY), {} + with zipfile.ZipFile(path) as z: + for info in z.infolist(): + assert info.compress_type == zipfile.ZIP_STORED, f"{path}: {info.filename} is compressed" + name, extra = struct.unpack(" tuple[int, int, int]: + fd, members = self.files[layer] + start, shape, dtype = members[name] + size = int(np.prod(shape[1:])) * dtype.itemsize + return fd, start + expert * size, size + + def read(self, layer: int, name: str, experts, pool=None) -> np.ndarray: + _, shape, dtype = self.files[layer][1][name] + host = np.empty((len(experts), *shape[1:]), dtype) + jobs = [(*self.slab(layer, name, e), host[s]) for s, e in enumerate(experts)] + for done in [pool.submit(Store.pread_into, *job) for job in jobs] if pool else jobs: + done.result() if pool else Store.pread_into(*done) + return host + + +class Experts(nn.Module): + """oMLX's routed ``Expert`` for the unfused path; ``indices`` are slots.""" + + quantizes_input = True + + def __init__(self, limit): + super().__init__() + self._limit = limit + + def __call__(self, x, indices, weights, sorted_indices=False, *, input_quantized=False): + # sorted: x [routes, 1, dim], indices [routes]; otherwise x [..., 1, 1, dim], indices [..., k] + lead = indices.shape if sorted_indices else indices.shape[:-1] + rows = x.reshape(*lead, x.shape[-1]) + gate = self.w1(rows, indices, sorted_indices=sorted_indices) + up = self.w3(rows, indices, sorted_indices=sorted_indices) + h = quantize_swiglu_activation(gate, up, weights, x.dtype, self._limit) + return self.w2(h, indices, sorted_indices=sorted_indices)[..., None, :].astype(x.dtype) + + +class Layer: + def __init__(self, index, moe, experts, weight, bias, order, slots): + assert weight.dtype == mx.bfloat16, weight.dtype # the gate kernel reads bf16 rows + self.index, self.gate, self.experts = index, moe.gate, experts + self.full_weight, self.full_bias = weight.astype(mx.float32), bias.astype(mx.float32) + mx.eval(self.full_weight, self.full_bias) + self.pool_bias = np.array(self.full_bias) + resident = [int(e) for e in order[:slots]] # hottest first + self.slot_expert = np.array(resident, dtype=np.int64) # -1: unrouted + self.slot_of = {e: s for s, e in enumerate(resident)} + self.last_use = -1.0 - np.arange(slots, dtype=np.float64) # colder starting experts go first + self.cooling: list[tuple[int, int]] = [] + self.free: list[int] = [] + self.reading: dict[int, tuple[int, list]] = {} + masked, slot_map = np.full(self.pool_bias.size, OFF, np.float32), np.zeros(self.pool_bias.size, np.uint32) + masked[resident], slot_map[resident] = self.pool_bias[resident], np.arange(slots) + self.slot_map = mx.array(slot_map) # expert -> slot, read only for experts whose bias is on + moe.gate.weight, moe.gate.bias, moe.experts = weight, mx.array(masked), experts + self.views = {} + for proj, field in TENSORS: + lin = getattr(experts, proj) + self.views[proj, field] = (lin, lin[field], np.asarray(memoryview(lin[field]))) + assert self.views_current() + + def views_current(self) -> bool: + """The host views still alias the arrays the kernels read.""" + return all(lin[field] is owner and np.frombuffer(memoryview(owner), np.uint8).ctypes.data == view.ctypes.data + for (_, field), (lin, owner, view) in self.views.items()) + + +class LatticeCache(Cache): + def __init__(self, model, config, root: Path, order, full, slots: int): + super().__init__(config, mx) + blocks = model.language_model.layers + self.codes = Codes(root, len(blocks)) + self.codebook = mx.array(e8p_packed_abs_grid()) + layers = [] + for i, blk in enumerate(blocks): + experts = Experts(config.swiglu_limit) + for proj in ("w1", "w3", "w2"): + codes, scales = (mx.array(self.codes.read(i, f"{proj}_{field}", order[i][:slots], self.pool)) + for field in ("codes", "scales")) + mx.eval(codes, scales) + setattr(experts, proj, QuantizedVQSwitchLinear( + input_dims=codes.shape[2] * 8, output_dims=codes.shape[1], num_experts=slots, codes=codes, + scales=scales, codebook=self.codebook, group_size=codes.shape[2] * 8 // scales.shape[2], + code_bits=16)) + layers.append(Layer(i, blk.ffn, experts, full[i, "weight"], full[i, "bias"], order[i], slots)) + self.pool_config = replace(config, n_routed_experts=layers[0].pool_bias.size) + self.ready(layers) + + def slabs(self, layer, expert, slot): + for (proj, field), (_, _, view) in layer.views.items(): + yield (*self.codes.slab(layer.index, f"{proj}_{field}", expert), view[slot]) + + def route(self, layer, expert, slot): + layer.slot_map[expert] = slot + layer.gate.bias[expert] = float(layer.pool_bias[expert]) + + def unroute(self, layer, expert, slot): + layer.gate.bias[expert] = OFF + + +def decode(cache, layer, moe, x, config, mma, stock_gate): + """One MoE for the decode step. x: [tokens, dim] bf16. Returns ([tokens, dim] bf16, slots, weights).""" + if layer.index == 0: + cache.begin() + nb, dim, inter, k = x.shape[0], config.dim, config.moe_inter_dim, config.n_activated_experts + scores, xq = stock_gate(moe, x, cache.pool_config) # every expert of the pool; act_quant(x) + slot, weight, want, want_score = _topk_kernel()( + inputs=[scores, moe.gate.bias, layer.full_bias, layer.slot_map, DM._const((config.route_scale,))], + template=[("NE", cache.pool_config.n_routed_experts), ("TOPK", k)], + grid=(nb * 32, 1, 1), threadgroup=(32, 1, 1), + output_shapes=[(nb * k,)] * 4, output_dtypes=[mx.uint32, mx.float32, mx.uint32, mx.float32]) + cache.pending.setdefault(cache.step, []).append((want, want_score)) # as evaluated: reading them later costs no eval + e, s = layer.experts, moe.shared_experts + # 16-bit activations: the lattice kernel is 1.7x slower on fp32 rows, and act_quant values are exact in bf16. + rows, slots = xq.reshape(nb, dim).astype(mx.bfloat16), slot.reshape(nb, k) + gate = e8p_m1_token_layout(rows, e.w1.codes, e.w1.scales, cache.codebook, slots, output_dims=inter) + up = e8p_m1_token_layout(rows, e.w3.codes, e.w3.scales, cache.codebook, slots, output_dims=inter) + h = quantize_swiglu_activation(gate, up, weight, mx.bfloat16, config.swiglu_limit) + down = e8p_m1_per_route(h.reshape(nb * k, inter), e.w2.codes, e.w2.scales, cache.codebook, slot, output_dims=dim) + routed = down.astype(mx.float32).reshape(nb, k, dim).sum(1).reshape(-1) # bf16 per expert, fp32 sum + # The shared expert on oMLX's kernels: its up kernel with no routed column (TOPK 0), then its + # shared down kernel, which adds the routed sum and rounds to bf16. + shared = tuple(DM.weight_format(getattr(s, n)) for n in ("w1", "w3", "w2")) + up_k, _, down_s = DM._kernels_nb(mma and all(f[0] in ("mxfp8", "affine") for f in shared), (DM.MXFP4,) * 3 + shared) + no_w, no_s, no_b = _unused(mx.uint32), _unused(mx.uint8), _unused(mx.float32) + hqs = up_k( + inputs=[no_b, no_b, no_b, DM._const((config.swiglu_limit,)), xq, no_w, no_s, no_b, no_w, no_s, no_b, + s.w1.weight, s.w1.scales, DM._biases(s.w1), s.w3.weight, s.w3.scales, DM._biases(s.w3)], + template=[("NE", 1), ("TOPK", 0), ("DIM", dim), ("INTER", inter), ("NB", nb)], + grid=(256, inter // 32, 1), threadgroup=(256, 1, 1), + output_shapes=[(1,), (nb * inter,), (1,), (1,)], + output_dtypes=[mx.float32, mx.float32, mx.int32, mx.float32])[1] + out = down_s( + inputs=[hqs, routed, s.w2.weight, s.w2.scales, DM._biases(s.w2)], + template=[("DIM", dim), ("INTER", inter), ("NB", nb)], + grid=(dim // 8 * 64, 1, 1), threadgroup=(64, 1, 1), + output_shapes=[(nb * dim,)], output_dtypes=[mx.bfloat16])[0] + return out.reshape(nb, dim), slots, weight.reshape(nb, k) + + +def hook(state: dict) -> None: + """Send the layers of ``state["cache"]`` through the lattice paths; every other MoE is oMLX's.""" + stock = SimpleNamespace(one=DM.fused_decode_moe, many=DM.fused_decode_moe_nb, supported=DM.supported, + gate=DM._gate, gate_call=language.Gate.__call__) + + def find(gate): + cache = state.get("cache") + return cache, cache.by_gate.get(id(gate)) if cache else None + + def supported(moe, x, config): + layer = find(moe.gate)[1] + return stock.supported(moe, x, config) if layer is None else x.size == config.dim and x.dtype == mx.bfloat16 + + def one(moe, x, config): + cache, layer = find(moe.gate) + if layer is None: + return stock.one(moe, x, config) + out, slots, weights = decode(cache, layer, moe, x.reshape(1, -1), config, False, stock.gate) + return out.reshape(x.shape), slots, weights + + def many(moe, x, config, mma=None): + cache, layer = find(moe.gate) + if layer is None: + return stock.many(moe, x, config, mma) + return decode(cache, layer, moe, x, config, DM.use_mma(x.shape[0]) if mma is None else mma, stock.gate) + + def gate_plain(self, x, image_mask): # prefill and every unfused path + cache, layer = find(self) + if layer is None: + return stock.gate_call(self, x, image_mask) + if layer.index == 0: + cache.begin() + index, weights = stock.gate_call(self, x, image_mask) # pool numbers of resident experts + return cache.observe(self, x, layer.slot_map[index]), weights + + DM.fused_decode_moe, DM.fused_decode_moe_nb, DM.supported = one, many, supported + language.Gate.__call__ = gate_plain + + +def patch(spec: dict, root: Path, slots: int) -> None: + full, state = {}, {} + compact_patch(spec, full) + load = loading.load + + def load_lattice(*args, **kwargs): + model, processor = load(*args, **kwargs) + config = model.language_model.layers[0].ffn.gate._config + state["cache"] = LatticeCache(model, config, root, spec["order"], full, slots) + full.clear() + return model, processor + + loading.load = load_lattice + hook(state) + + +def check(config_path: Path, root: Path, slots: int = 32, chain: int = 40) -> None: + from bench_expert_formats import median_ms, omlx_moe + from omlx.patches.deepseek_v41.config import ModelConfig + + rng = np.random.default_rng(0) + c = replace(ModelConfig.from_dict(json.loads(config_path.read_text())), n_routed_experts=64) + moes = [omlx_moe(c, "mxfp4", 4, 32, rng)[0] for _ in range(2)] + for moe in moes: # shared experts: random MXFP8 bytes below the codes for inf and NaN + for name in ("w1", "w3", "w2"): + shape = getattr(moe.shared_experts, name).weight.shape + codes = rng.integers(0, 0x60, size=(*shape, 4), dtype=np.uint8) | (rng.integers(0, 2, size=(*shape, 4), dtype=np.uint8) << 7) + getattr(moe.shared_experts, name).weight = mx.array(codes.view(np.uint32).reshape(shape)) + pool = 192 + full = {} + for i in range(2): + full[i, "weight"] = mx.array(rng.normal(size=(pool, c.dim)).astype(np.float32)).astype(mx.bfloat16) + full[i, "bias"] = mx.array(rng.normal(size=(pool,)).astype(np.float32) * 0.1) + os.environ.setdefault("OMLX_ROUTED_TAU", "inf") + model = SimpleNamespace(language_model=SimpleNamespace(layers=[SimpleNamespace(ffn=m) for m in moes])) + state = {"cache": LatticeCache(model, c, root, [rng.permutation(pool).tolist() for _ in moes], full, slots)} + cache = state["cache"] + hook(state) + + def token(): + return mx.array(rng.normal(size=(c.dim,)).astype(np.float32)).astype(mx.bfloat16) + + def dense(layer, proj, expert): + return decode_weight_matrix(*(cache.codes.read(layer, f"{proj}_{f}", [expert])[0] for f in ("codes", "scales")), + code_bits=16) + + def host(a): + return np.array(a.astype(mx.float32)) + + def reference(i, x, slots_, weights): + """The same MoE from matrices decoded on the host (and oMLX's shared expert, unfused), + rounded to bf16 where oMLX's kernels round.""" + xq, limit, total = host(quantize_activation(x.reshape(1, -1)))[0], c.swiglu_limit, 0.0 + for slot, weight in zip(slots_, weights): + expert = int(cache.layers[i].slot_expert[slot]) + g, u = bf16(dense(i, "w1", expert) @ xq), bf16(dense(i, "w3", expert) @ xq) + if limit: + g, u = np.minimum(g, limit), np.clip(u, -limit, limit) + h = bf16(g / (1 + np.exp(-g)) * u * weight) + total = total + bf16(dense(i, "w2", expert) @ host(quantize_activation(mx.array(h[None])))[0]) + shared = moes[i].shared_experts(quantize_activation(x.reshape(1, 1, -1)), input_quantized=True) + return total + host(shared).reshape(-1) + + def bf16(a): + return host(mx.array(a.astype(np.float32)).astype(mx.bfloat16)) + + def gap(a, b): + return float(np.max(np.abs(a - b)) / np.max(np.abs(b))) + + def against_reference(i): + x = token() + out, slots_, weights = DM.fused_decode_moe(moes[i], x.reshape(1, 1, -1), c) + return gap(host(out).reshape(-1), reference(i, x, np.array(slots_).reshape(-1), np.array(weights).reshape(-1))) + + worst = max(against_reference(i) for i in (0, 1) for _ in range(2)) + print(f"decode step against host-decoded matrices: worst relative gap {worst:.4f}") + assert worst < 2e-2 + + x = mx.stack([token() for _ in range(40)])[None] # [1, 40, dim]: the sorted-route prefill path + decoded = host(DM.fused_decode_moe_nb(moes[0], x[0, :8], c)[0]) + worst = max(gap(host(moes[0](x[:, :t], None))[0, :8][: min(t, 8)], decoded[: min(t, 8)]) for t in (40, 4)) + print(f"prefill path against decode path: worst relative gap {worst:.4f}") + assert worst < 2e-2 + + cache.tau = 0.0 # admit everything demanded + for step in range(24): + for moe in moes: + mx.eval(DM.fused_decode_moe(moe, token().reshape(1, 1, -1), c)[0]) + time.sleep(0.05) + print("after 24 tokens at tau 0:", {k: cache.stats[k] for k in ("wanted", "admitted", "installed", "evicted", "no_slot")}) + time.sleep(0.5) + assert cache.stats["installed"] > 0 and cache.mismatched() == 0 + cache.tau = float("inf") + worst = max(against_reference(i) for i in (0, 1) for _ in range(3)) + print(f"after installs, against host-decoded matrices: worst relative gap {worst:.4f}; no slot differs from its bytes") + assert worst < 2e-2 + + state_x = {"x": token().reshape(1, 1, -1)} + + def step(n): + x = state_x["x"] + for j in range(n): + out = DM.fused_decode_moe(moes[j % 2], x, c)[0].astype(mx.float32) + x = (out / (mx.max(mx.abs(out)) + 1e-6) * 3).astype(mx.bfloat16) + mx.eval(x) + state_x["x"] = x + + print(f"decode-step MoE (router over {pool}, {c.n_activated_experts} lattice experts, shared expert): " + f"{median_ms(step, chain):.3f} ms") + + +if __name__ == "__main__": + if sys.argv[1:2] == ["check"]: + import argparse + + p = argparse.ArgumentParser() + p.add_argument("--config", type=Path, required=True) + p.add_argument("--codes", type=Path, required=True) + a = p.parse_args(sys.argv[2:]) + check(a.config.expanduser(), a.codes.expanduser()) + else: + patch(json.loads(Path(os.environ["OMLX_COMPACT"]).read_text()), Path(os.environ["OMLX_LATTICE"]).expanduser(), + int(os.environ.get("OMLX_LATTICE_SLOTS", "144"))) + from omlx.cli import main + + sys.exit(main()) diff --git a/scripts/v41_build/omlx_routed.py b/scripts/v41_build/omlx_routed.py index 4b07ca8..34dcde6 100644 --- a/scripts/v41_build/omlx_routed.py +++ b/scripts/v41_build/omlx_routed.py @@ -58,9 +58,6 @@ def slab(self, key: str) -> tuple[int, int, int]: begin, end = header[key]["data_offsets"] return fd, start + begin, end - begin - def read_into(self, key: str, view: np.ndarray) -> int: - return self.pread_into(*self.slab(key), view) - @staticmethod def pread_into(fd: int, offset: int, nbytes: int, view: np.ndarray) -> int: """Fill ``view`` from the file. Positional, so reader threads may call it.""" @@ -104,38 +101,48 @@ def views_current(self) -> bool: class Cache: - def __init__(self, model, config, spec, full, mx): + """The policy. A subclass sets ``layers`` and says where a slot's bytes live (``slabs``) and + how the router switches an expert on and off (``route``, ``unroute``).""" + + def __init__(self, config, mx): self.mx, self.config = mx, config self.tau = float(os.environ.get("OMLX_ROUTED_TAU", "0.2")) self.spare = int(os.environ.get("OMLX_ROUTED_SPARE", "4")) self.lag = int(os.environ.get("OMLX_ROUTED_LAG", "2")) self.log_path = os.environ.get("OMLX_ROUTED_LOG") assert config.score_func == "sqrtsoftplus" and self.lag >= 1 - self.store = Store(Path(spec["source"])) self.pool = ThreadPoolExecutor(max_workers=8, thread_name_prefix="routed-io") - blocks = model.language_model.layers - self.layers = [Layer(i, blk.ffn, full[i, "weight"], full[i, "bias"], spec["hot"][i], spec["rank"][i], mx) - for i, blk in enumerate(blocks)] - self.by_gate = {id(layer.gate): layer for layer in self.layers} self.step = 0 self.pending: dict[int, list] = {} # forward call -> per-layer (index, weight) demand arrays self.stats = dict.fromkeys(("calls", "tokens", "demand", "resident", "wanted", "admitted", "installed", - "evicted", "bytes", "no_slot"), 0) + "evicted", "bytes", "no_slot", "policy_s"), 0) self.started = time.time() - for key in {name: key for key, name in self.store.index.items() if ".ffn.experts." in key}.values(): - self.store.slab(key) # open every expert shard now, not during a token - self.check_layout() - - def check_layout(self): - """A resident expert's rows must equal its stored bytes: installs rely on it.""" - for layer in self.layers[:: max(1, len(self.layers) // 4)]: - expert = int(layer.slot_expert[0]) - for proj in PROJECTIONS: - for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): - key = f"layers.{layer.index}.ffn.experts.{expert}.{proj}.{field}" - stored = np.empty(view[0].nbytes, np.uint8) - self.store.read_into(key, stored) - assert np.array_equal(stored, view[0].reshape(-1).view(np.uint8)), f"layout differs: {key}" + + def ready(self, layers): + self.layers = layers + self.by_gate = {id(layer.gate): layer for layer in layers} + # A resident expert's rows must equal its stored bytes: installs rely on it. + for layer in layers[:: max(1, len(layers) // 4)]: + assert not self.differing(layer, 0), f"layer {layer.index}: slot 0 differs from its stored bytes" + + def slabs(self, layer, expert, slot): + """(fd, offset, nbytes, the slot's host view) for each stored tensor of ``expert``.""" + raise NotImplementedError + + def route(self, layer, expert, slot): + raise NotImplementedError + + def unroute(self, layer, expert, slot): + raise NotImplementedError + + def differing(self, layer, slot) -> int: + """Tensors of a routed slot whose bytes differ from its expert's stored bytes.""" + bad = 0 + for fd, offset, nbytes, view in self.slabs(layer, int(layer.slot_expert[slot]), slot): + stored = np.empty(nbytes, np.uint8) + Store.pread_into(fd, offset, nbytes, stored) + bad += not np.array_equal(stored, view.reshape(-1).view(np.uint8)) + return bad # --- demand, recorded while a forward call is built ------------------------------------- @@ -161,7 +168,9 @@ def begin(self): self.step += 1 ready = self.step - self.lag if self.tau != float("inf") and ready in self.pending: + started = time.perf_counter() self.harvest(ready, self.pending.pop(ready)) + self.stats["policy_s"] += time.perf_counter() - started for old in [s for s in self.pending if s < ready]: del self.pending[old] if self.log_path and self.step % 64 == 0: @@ -172,7 +181,8 @@ def harvest(self, call, demand): self.stats["calls"] += 1 # Reading this call's demand means every earlier call has left the GPU. for layer, (index, weight) in zip(self.layers, demand): - index, weight = np.asarray(index), np.asarray(weight, dtype=np.float64) + index = np.asarray(index).reshape(-1, c.n_activated_experts) + weight = np.asarray(weight, dtype=np.float64).reshape(index.shape) weight = weight / (weight.sum(-1, keepdims=True) + 1e-20) * c.route_scale if layer.index == 0: self.stats["tokens"] += index.shape[0] @@ -200,11 +210,7 @@ def harvest(self, call, demand): self.stats["no_slot"] += 1 continue slot = layer.free.pop() - futures = [] - for proj in PROJECTIONS: - base = f"layers.{layer.index}.ffn.experts.{expert}.{proj}." - for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): - futures.append(self.pool.submit(Store.pread_into, *self.store.slab(base + field), view[slot])) + futures = [self.pool.submit(Store.pread_into, *slab) for slab in self.slabs(layer, expert, slot)] layer.reading[expert] = (slot, futures) self.stats["admitted"] += 1 self.evict(layer, set(wanted)) @@ -217,8 +223,7 @@ def install(self, layer) -> None: for expert in done: slot, futures = layer.reading.pop(expert) self.stats["bytes"] += sum(f.result() for f in futures) - layer.gate.weight[slot] = layer.typed_weight[expert] - layer.gate.bias[slot] = layer.typed_bias[expert] + self.route(layer, expert, slot) layer.slot_expert[slot], layer.slot_of[expert] = expert, slot layer.last_use[slot] = self.step self.stats["installed"] += 1 @@ -231,10 +236,11 @@ def evict(self, layer, protected) -> None: order = np.argsort(layer.last_use, kind="stable") victims = [int(s) for s in order if layer.slot_expert[s] >= 0 and int(layer.slot_expert[s]) not in protected] for slot in victims[:short]: - del layer.slot_of[int(layer.slot_expert[slot])] + expert = int(layer.slot_expert[slot]) + del layer.slot_of[expert] layer.slot_expert[slot] = -1 layer.last_use[slot] = np.inf - layer.gate.bias[slot] = OFF + self.unroute(layer, expert, slot) # Calls built before this one may still route here; its bytes may change only once # the call being built now is known to have finished. layer.cooling.append((slot, self.step)) @@ -242,15 +248,8 @@ def evict(self, layer, protected) -> None: def mismatched(self) -> int: """Routed slots (first and last layer) whose rows differ from their expert's stored bytes.""" - bad = 0 - for layer in (self.layers[0], self.layers[-1]): - for slot, expert in enumerate(layer.slot_expert.tolist()): - for proj in PROJECTIONS if expert >= 0 else (): - for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): - stored = np.empty(view[slot].nbytes, np.uint8) - self.store.read_into(f"layers.{layer.index}.ffn.experts.{expert}.{proj}.{field}", stored) - bad += not np.array_equal(stored, view[slot].reshape(-1).view(np.uint8)) - return bad + return sum(self.differing(layer, slot) for layer in (self.layers[0], self.layers[-1]) + for slot, expert in enumerate(layer.slot_expert.tolist()) if expert >= 0) def write_log(self): row = {"t": round(time.time() - self.started, 2), "step": self.step, **self.stats, @@ -261,6 +260,31 @@ def write_log(self): f.write(json.dumps(row) + "\n") +class FP4Cache(Cache): + """Slots are rows of oMLX's stacked MXFP4 arrays; the router holds one row per slot.""" + + def __init__(self, model, config, spec, full, mx): + super().__init__(config, mx) + self.store = Store(Path(spec["source"])) + for key in {name: key for key, name in self.store.index.items() if ".ffn.experts." in key}.values(): + self.store.slab(key) # open every expert shard now, not during a token + self.ready([Layer(i, blk.ffn, full[i, "weight"], full[i, "bias"], spec["hot"][i], spec["rank"][i], mx) + for i, blk in enumerate(model.language_model.layers)]) + + def slabs(self, layer, expert, slot): + for proj in PROJECTIONS: + base = f"layers.{layer.index}.ffn.experts.{expert}.{proj}." + for field, view in (("weight", layer.views[proj][3]), ("scale", layer.views[proj][4])): + yield (*self.store.slab(base + field), view[slot]) + + def route(self, layer, expert, slot): + layer.gate.weight[slot] = layer.typed_weight[expert] + layer.gate.bias[slot] = layer.typed_bias[expert] + + def unroute(self, layer, expert, slot): + layer.gate.bias[slot] = OFF + + def patch(spec: dict) -> None: import mlx.core as mx from omlx.patches.deepseek_v41 import decode_moe, language, loading @@ -272,7 +296,7 @@ def patch(spec: dict) -> None: def load_routed(*args, **kwargs): model, processor = load(*args, **kwargs) - state["cache"] = Cache(model, model.language_model.layers[0].ffn.gate._config, spec, full, mx) + state["cache"] = FP4Cache(model, model.language_model.layers[0].ffn.gate._config, spec, full, mx) full.clear() return model, processor From 3e620bde017e305dc1eacf18114bf971b4dbae94 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:02:13 -0800 Subject: [PATCH 53/74] docs: the exact pruned pool of 192 also fails with thinking on; not the 2-bit codes or the cache --- .../2026-11-08-v41-reap-recovery-log.md | 11 +++++++++++ .../2026-11-10-v41-wow-build-plan.md | 10 +++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index c6abdb8..d4fddc3 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -927,3 +927,14 @@ What it says: codes are not yet separated from pruning. - Rubric item 6 moves from 5 to 7. Total 50. Server stopped; nothing is running. +### 2026-11-10: the exact pruned model also fails with thinking on + +- The pool of 192 at FP4 under stock oMLX offload (exact, 7.1 tok/s), same prompt, greedy, + thinking on: 4,096 reasoning tokens and no answer, repeating itself from about character + 10,500 of 16,235. The lattice cache repeats from about 5,300; the fixed lattice set and the + sampled run do not repeat, and none of the four answers. +- So the failure with thinking on is not caused by 2-bit codes or by the cache. It is in the + pruned checkpoint at full precision, or in the model, template or greedy decoding as served + here; the unpruned model has not been run on this prompt. +- Rubric unchanged at 50. Server stopped; nothing is running. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index e721a05..930f204 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -469,9 +469,13 @@ Text, same prompt as before (a script that finds duplicate files by SHA-256): - Thinking on: no answer within 4,096 tokens in any of three runs. Greedy with the cache on is coherent for about 1,000 tokens and then repeats itself; default sampling does not repeat but keeps reviewing code it drafted in its reasoning; greedy with the policy off (fixed 144) - also runs out. So the cache is not the cause. The exact pool of 192 at FP4 starts writing - after about 500 characters of reasoning on this prompt (K1's 1,024-token run); it was not - run to 4,096 tokens, so 2-bit codes and pruning are not separated here. + also runs out. So the cache is not the cause. +- The exact pool of 192 at FP4 (stock oMLX offload, 7.1 tok/s, greedy) fails the same way: 4,096 + reasoning tokens, no answer, and from about character 10,500 of 16,235 it repeats the same + list of options it will not add. The lattice cache starts repeating at about character 5,300; + the fixed lattice set and the sampled run do not repeat. So neither the 2-bit codes nor the + cache causes the failure with thinking on: the pruned checkpoint has it at full precision. + Whether the unpruned model has it too is not known; that model is not on the Mac. - With the cache on, greedy output is not reproducible: install timing changes routing. Two runs diverged after 18 characters; a third matched the second. From b8b218da1c001efe007fbc3d326b6bc4a25b5601 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:14:23 -0800 Subject: [PATCH 54/74] docs: thinking finishes under sampling with a 12,000-token budget (lattice cache, correct script, 20.7 tok/s) --- .../2026-11-08-v41-reap-recovery-log.md | 10 ++++++++++ .../deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index d4fddc3..182da1a 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -938,3 +938,13 @@ What it says: here; the unpruned model has not been run on this prompt. - Rubric unchanged at 50. Server stopped; nothing is running. +### 2026-11-10: thinking finishes under sampling, given 12,000 tokens + +- Lattice cache, 144 residents, thinking on, default sampling, budget 12,000: 7,567 reasoning + tokens with no repeated sentence, then a 620-token answer whose script runs correctly. + 400 s, 20.7 tok/s over 8,187 tokens, 62 GiB. +- So with thinking on the model is slow to decide rather than broken, and greedy decoding is + what loops (on the exact pruned model too). Speed at 8,000 tokens of context is a quarter + below the 1,000-token figure; unattributed. +- Rubric unchanged at 50. Server stopped; nothing is running. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 930f204..85fcc96 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -476,6 +476,12 @@ Text, same prompt as before (a script that finds duplicate files by SHA-256): the fixed lattice set and the sampled run do not repeat. So neither the 2-bit codes nor the cache causes the failure with thinking on: the pruned checkpoint has it at full precision. Whether the unpruned model has it too is not known; that model is not on the Mac. +- Given room, thinking does finish under default sampling: the lattice cache with a budget of + 12,000 tokens reasoned for 7,567 tokens without repeating itself, then answered in 620 with + a script that runs correctly on the test tree. 400 s in all, 20.7 tok/s over the 8,187 + tokens (1.5 installs per token, policy 2.4 ms per token). The drop from 26–28 tok/s at + 1,000 tokens is not attributed; context length is the first suspect and the fixed set was + never timed this far out. - With the cache on, greedy output is not reproducible: install timing changes routing. Two runs diverged after 18 characters; a third matched the second. From ef2877856a92a56c6f1f41ac6d84333ddb9f45a2 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:26:52 -0800 Subject: [PATCH 55/74] v41_reap: the E8P fit in torch (matches the Metal search) and a pod job for the 192 removed experts --- .../2026-11-08-v41-reap-recovery-log.md | 12 + scripts/runpod/v41_reap/keep_quant_torch.py | 252 ++++++++++++++++++ scripts/runpod/v41_reap/pool384_pod.sh | 55 ++++ 3 files changed, 319 insertions(+) create mode 100644 scripts/runpod/v41_reap/keep_quant_torch.py create mode 100755 scripts/runpod/v41_reap/pool384_pod.sh diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 182da1a..0a6d527 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -948,3 +948,15 @@ What it says: below the 1,000-token figure; unattributed. - Rubric unchanged at 50. Server stopped; nothing is running. +### 2026-11-10: pool of 384, fit prepared, no GPU in the volume's data center + +- The lattice fit had no CUDA path (it ran on the Mac's Metal search). `keep_quant_torch.py` is + the same fit in torch; on 64 rows of nine projections it reproduces the stored codes + (99.99–100% equal, same weighted error). `pool384_pod.sh` runs it on a pod for the 192 + experts per layer that REAP-50 removed and extracts all 384 router rows. +- Importance for the removed experts is built (`imp-pruned.npz`, 228 MB): 17 experts that + calibration never routed get their layer's mean importance. +- No GPU type had stock in US-NE-1, where the volume is; one create of 1×H100 failed on + capacity. Nothing was created and nothing is billing beyond the volume. +- Rubric unchanged at 50. + diff --git a/scripts/runpod/v41_reap/keep_quant_torch.py b/scripts/runpod/v41_reap/keep_quant_torch.py new file mode 100644 index 0000000..f02142d --- /dev/null +++ b/scripts/runpod/v41_reap/keep_quant_torch.py @@ -0,0 +1,252 @@ +"""``keep_quant.py``'s 2-bit E8P fit on torch, for the experts a REAP prune removed. + +``keep_quant.py`` fits the survivors of a pruned checkpoint with KEEP's Metal search, on a Mac. +This fits the other experts from the full checkpoint on a CUDA pod, so the cache's pool can be +all 384. The fit is the same alternation as ``quantize_weight_importance_aware`` with its +default policy (weighted nearest E8P codeword under the per-input-channel importance, then the +exact weighted least-squares group scale; 16-bit codes; group 512 for w1/w3, 256 for w2), in +torch. Rows are independent, so ``check`` can compare a few rows against a ``keep_quant`` file. + + # Mac: the importance of the removed experts, one small file to upload + keep_quant_torch.py importance --stats STATS --survivors survivors-192.json --out imp-pruned.npz + # Mac: the torch fit against stored codes of the survivors (fitted by the Metal search) + keep_quant_torch.py check --hf-path PRUNED_CKPT --survivors survivors-192.json --stats STATS --codes QUANT_DIR + # pod: fit, one process per GPU (--layers splits the work); finished layers are skipped + keep_quant_torch.py fit --hf-path FULL_CKPT --importance imp-pruned.npz --out OUT [--layers 0,1,...] + # pod: every router row of the full checkpoint + keep_quant_torch.py router --hf-path FULL_CKPT --out router-384.npz + +``fit`` writes ``layer-XX.npz`` in ``keep_quant``'s format (experts in the order of +``imp-pruned.npz``'s ``experts`` row, which ``experts.json`` repeats) and ``metrics.jsonl``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import numpy as np +import torch + +KIT = Path(__file__).resolve().parent +sys.path.insert(0, str(KIT)) +sys.path.insert(0, str(KIT.parents[2] / "src")) + +from keep.vq.e8 import E8P_SHUFFLE_MAP, e8p_abs_grid, e8p_full_grid + +PROJ_SPACE = {"w1": "hidden", "w3": "hidden", "w2": "down"} + + +def _parity(x: torch.Tensor) -> torch.Tensor: + x = x ^ (x >> 4) + x = x ^ (x >> 2) + return (x ^ (x >> 1)) & 1 + + +class Search: + """Weighted nearest E8P codeword; ``e8._encode_e8p_rtn_per_row_diagonal_hessian`` in torch.""" + + def __init__(self, device, vectors_per_chunk: int = 1 << 17): + base = e8p_abs_grid().astype(np.float32) # signed base rows [256, 8] + self.abs_rows = torch.from_numpy(np.abs(base)).to(device) + self.bit = torch.from_numpy(E8P_SHUFFLE_MAP.astype(np.int32)).to(device) # dim -> sign bit + self.base_signs = ((torch.from_numpy(base < 0).to(device).int() << self.bit).sum(-1)).int() + self.grid = torch.from_numpy(e8p_full_grid().astype(np.float32)).to(device) # [65536, 8] + self.vectors_per_chunk = vectors_per_chunk + + def assign(self, vectors: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """vectors [rows, P, 8] (already divided by their scales), w [P, 8] -> codes [rows, P] int32.""" + abs_norm_w = w @ (self.abs_rows * self.abs_rows).T # [P, 256] + step = max(1, self.vectors_per_chunk // vectors.shape[1]) + return torch.cat([self._assign(vectors[i:i + step], w, abs_norm_w) for i in range(0, vectors.shape[0], step)]) + + def _assign(self, chunk, w, abs_norm_w): + best_dist = torch.full(chunk.shape[:2], torch.inf, device=chunk.device) + best_code = torch.zeros(chunk.shape[:2], dtype=torch.int32, device=chunk.device) + for parity, shift in ((0, 0.25), (1, -0.25)): + target = chunk - shift + weighted = target.abs() * w # [R, P, 8] + dist = (target.abs() * weighted).sum(-1, keepdim=True) + abs_norm_w - 2.0 * (weighted @ self.abs_rows.T) + signs = ((target < 0).int() << self.bit).sum(-1, dtype=torch.int32)[..., None] ^ self.base_signs # [R, P, 256] + odd = _parity(signs).bool() + # An odd sign count is fixed by flipping the dimension that costs least. + dist = torch.where(odd, dist + (4 * weighted[..., None, :] * self.abs_rows).amin(-1), dist) + row = dist.argmin(-1, keepdim=True) + row_dist, signs, odd = dist.gather(-1, row)[..., 0], signs.gather(-1, row)[..., 0], odd.gather(-1, row)[..., 0] + row = row[..., 0] + penalty = 4 * weighted * self.abs_rows[row] # [R, P, 8] + flipped = signs[..., None] ^ (1 << self.bit) + tied = penalty == penalty.amin(-1, keepdim=True) + flip = torch.where(tied, flipped, 65535).argmin(-1) # ties: the lowest resulting code + signs = torch.where(odd, signs ^ (1 << self.bit[flip]), signs) + code = (row.int() << 8) | (signs ^ parity) + take = (row_dist < best_dist) | ((row_dist == best_dist) & (code < best_code)) + best_dist, best_code = torch.where(take, row_dist, best_dist), torch.where(take, code, best_code) + return best_code + + def fit(self, weight: torch.Tensor, importance: torch.Tensor, group: int, iterations: int = 8): + """weight [out, in] fp32, importance [in] -> (codes uint16 [out, in/8], scales fp16 [out, in/group], + weighted relative squared error of the stored code).""" + out_dim, in_dim = weight.shape + grouped = weight.reshape(out_dim, in_dim // group, group // 8, 8) + h = importance.reshape(in_dim // group, group // 8, 8) + w = importance.reshape(-1, 8) + scales = grouped.abs().amax((2, 3)) / self.grid.abs().max() + scales = torch.where(scales > 0, scales, torch.ones_like(scales)) + + def assign(): + return self.assign((grouped / scales[:, :, None, None]).reshape(out_dim, -1, 8), w).reshape(grouped.shape[:3]) + + for _ in range(iterations): + decoded = self.grid[assign().long()] + numerator = (h * grouped * decoded).double().sum((2, 3)) + denominator = (h * decoded * decoded).double().sum((2, 3)) + ok = (denominator > 0) & (numerator > 0) + scales = torch.where(ok, numerator / denominator.clamp_min(1e-300), scales.double()).float() + codes = assign() + stored = scales.half() + error = self.grid[codes.long()] * stored.float()[:, :, None, None] - grouped + wrel = float((h * error * error).double().sum() / (h * grouped * grouped).double().sum()) + return codes.reshape(out_dim, in_dim // 8).cpu().numpy().astype(np.uint16), stored.cpu().numpy(), wrel + + +def decode_fp4(weight: torch.Tensor, scale: torch.Tensor, device) -> torch.Tensor: + import torch_kernel + + # The packed dtypes leave the CPU as bytes and floats; not every backend has them. + w = torch_kernel.decode_e2m1(weight.contiguous().view(torch.uint8).to(device)) # [out, in] fp32 + return (w.unflatten(-1, (-1, 32)) * scale.float().to(device)[..., None]).flatten(-2) + + +def load_expert(ckpt, layer: int, expert: int, device) -> dict[str, torch.Tensor]: + sd = ckpt.load(f"layers.{layer}.ffn.experts.{expert}.") + return {proj: decode_fp4(sd[f"{proj}.weight"], sd[f"{proj}.scale"], device) for proj in PROJ_SPACE} + + +def importance(args) -> None: + from keep_quant import pod_importance + + survivors = json.loads(args.survivors.read_text())["survivors"] + full = pod_importance(args.stats) # {space: [L, 384, in]} + n_layers, n_experts = full["hidden"].shape[:2] + removed = np.array([sorted(set(range(n_experts)) - set(survivors[str(layer)])) for layer in range(n_layers)]) + out, replaced = {"experts": removed}, 0 + for space, total in full.items(): + picked = np.stack([total[layer][removed[layer]] for layer in range(n_layers)]) + # The search needs some weight in every group of eight inputs. An expert that calibration + # never routed has none: it gets its layer's mean over all experts. + dead = ~(picked.reshape(*picked.shape[:2], -1, 8) > 0).any(-1).all(-1) # [L, E] + picked[dead] = np.broadcast_to(total.mean(1)[:, None, :], picked.shape)[dead] + replaced += int(dead.sum()) + assert (picked.reshape(*picked.shape[:2], -1, 8) > 0).any(-1).all() and np.isfinite(picked).all(), space + out[space] = picked.astype(np.float32) + np.savez(args.out, **out) + print(f"{args.out}: {removed.shape[1]} removed experts in each of {n_layers} layers; " + f"{replaced} (expert, space) importances replaced by the layer mean") + + +def check(args) -> None: + """A few rows of some surviving experts, fitted here, against the stored Metal-search codes.""" + import stream + from keep_quant import pod_importance + + device = torch.device(args.device) + survivors = json.loads(args.survivors.read_text())["survivors"] + full, ckpt, search = pod_importance(args.stats), stream.HFCheckpoint(args.hf_path), Search(device) + grid = e8p_full_grid() + for layer, expert in ((0, 3), (20, 0), (39, 191)): + z = np.load(args.codes / f"layer-{layer:02d}.npz") + weights = load_expert(ckpt, layer, expert, device) + original = sorted(survivors[str(layer)])[expert] + for proj, space in PROJ_SPACE.items(): + w = weights[proj][: args.rows] + d = torch.from_numpy(full[space][layer][original].astype(np.float32)).to(device) + started = time.time() + codes, scales, wrel = search.fit(w, d, 512 if w.shape[1] % 512 == 0 else 256) + seconds = time.time() - started + ref_codes, ref_scales = z[f"{proj}_codes"][expert, : args.rows], z[f"{proj}_scales"][expert, : args.rows] + group = w.shape[1] // ref_scales.shape[1] + ref = (grid[ref_codes].reshape(args.rows, -1, group // 8, 8) * ref_scales.astype(np.float32)[:, :, None, None]) + ref = torch.from_numpy(ref.reshape(args.rows, -1)).to(device) + ref_wrel = float((d * (ref - w) ** 2).double().sum() / (d * w * w).double().sum()) + same = float((codes == ref_codes).mean()) + print(f"layer {layer} expert {expert} {proj}: codes equal {same:.4f}, weighted error {wrel:.6f} " + f"(stored {ref_wrel:.6f}), {seconds:.2f} s for {args.rows} rows") + assert same > 0.99 and abs(wrel - ref_wrel) < 1e-3 * ref_wrel, (same, wrel, ref_wrel) + + +def fit(args) -> None: + import stream + + device = torch.device(args.device) + with np.load(args.importance) as z: + imp = {"hidden": z["hidden"], "down": z["down"]} + experts = z["experts"] + ckpt, search = stream.HFCheckpoint(args.hf_path), Search(device) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "experts.json").write_text(json.dumps({"experts": experts.tolist(), "source": str(args.hf_path)})) + layers = range(experts.shape[0]) if args.layers == "all" else [int(x) for x in args.layers.split(",")] + for layer in layers: + dest = args.out / f"layer-{layer:02d}.npz" + if dest.exists(): + continue + started = time.time() + arrays: dict[str, list[np.ndarray]] = {} + errors: dict[str, list[float]] = {proj: [] for proj in PROJ_SPACE} + for slot, expert in enumerate(experts[layer][: args.experts or None].tolist()): + weights = load_expert(ckpt, layer, expert, device) + for proj, space in PROJ_SPACE.items(): + w = weights[proj] + codes, scales, wrel = search.fit(w, torch.from_numpy(imp[space][layer, slot]).to(device), + 512 if w.shape[1] % 512 == 0 else 256, args.iterations) + arrays.setdefault(f"{proj}_codes", []).append(codes) + arrays.setdefault(f"{proj}_scales", []).append(scales) + errors[proj].append(wrel) + tmp = dest.with_suffix(".tmp.npz") + np.savez(tmp, code_bits=np.asarray(16), **{k: np.stack(v) for k, v in arrays.items()}) + tmp.rename(dest) + row = {"layer": layer, "experts": len(errors["w1"]), "seconds": round(time.time() - started, 1), + **{f"{proj}_wrel_mse": float(np.mean(v)) for proj, v in errors.items()}} + with (args.out / "metrics.jsonl").open("a") as f: + f.write(json.dumps(row) + "\n") + print(json.dumps(row), flush=True) + + +def router(args) -> None: + import stream + + ckpt, rows = stream.HFCheckpoint(args.hf_path), {} + for layer in range(args.n_layers): + for name, tensor in ckpt.load(f"layers.{layer}.ffn.gate.").items(): + rows.setdefault(name, []).append(tensor.float().numpy()) # bf16 widens to fp32 without loss + np.savez(args.out, **{name: np.stack(v) for name, v in rows.items()}) + print({name: np.stack(v).shape for name, v in rows.items()}) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="command", required=True) + for name in ("importance", "check", "fit", "router"): + s = sub.add_parser(name) + s.add_argument("--hf-path", type=Path) + s.add_argument("--survivors", type=Path) + s.add_argument("--stats", type=Path) + s.add_argument("--codes", type=Path) + s.add_argument("--importance", type=Path) + s.add_argument("--out", type=Path) + s.add_argument("--layers", default="all") + s.add_argument("--iterations", type=int, default=8) + s.add_argument("--experts", type=int, default=0, help="debug: only the first N experts of a layer") + s.add_argument("--rows", type=int, default=64) + s.add_argument("--n-layers", type=int, default=40) + s.add_argument("--device", default="cuda" if torch.cuda.is_available() else "mps") + args = p.parse_args() + with torch.no_grad(): + {"importance": importance, "check": check, "fit": fit, "router": router}[args.command](args) + + +if __name__ == "__main__": + main() diff --git a/scripts/runpod/v41_reap/pool384_pod.sh b/scripts/runpod/v41_reap/pool384_pod.sh new file mode 100755 index 0000000..e0ee904 --- /dev/null +++ b/scripts/runpod/v41_reap/pool384_pod.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Pool of 384 on one GPU pod: 2-bit E8P codes for the 192 experts per layer that REAP-50 removed, +# and every router row of the full checkpoint. +# +# The volume already holds what this needs, so the GPU starts work after one copy: +# /workspace/v41-hf the pinned full checkpoint +# /workspace/keep this repo (keep_quant_torch.py uploaded into the kit) +# /workspace/pool384/imp-pruned.npz importance of the removed experts (built on the Mac) +# Output, on the volume: /workspace/pool384/e8p-podimp-pruned/layer-XX.npz, router-384.npz, DONE. +set -euo pipefail +W=/workspace +P=$W/pool384 +KIT=$W/keep/scripts/runpod/v41_reap +LOCAL=${LOCAL:-/root/v41-experts} +export PYTHONPATH=$W/keep/src PIP_BREAK_SYSTEM_PACKAGES=1 +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$P/pod.log"; } +trap 'status=$?; [[ $status -ne 0 ]] && { log "FAILED (status $status)"; touch "$P/FAILED"; }' EXIT + +# Never read the checkpoint from the volume (one stream is 120 MB/s). Copy the shards that hold +# routed experts or router rows to the container's disk with 16 streams. +log "copy expert and router shards to $LOCAL" +mkdir -p "$LOCAL" +python3 - "$W/v41-hf" > /tmp/shards.txt <<'PY' +import json, sys +names = json.load(open(sys.argv[1] + "/model.safetensors.index.json"))["weight_map"] +print("\n".join(sorted({shard for name, shard in names.items() if ".ffn.experts." in name or ".ffn.gate." in name}))) +PY +(cd "$W/v41-hf" && xargs -P 16 -I{} cp {} "$LOCAL/" < /tmp/shards.txt && cp ./*.json "$LOCAL/") +want=$(cd "$W/v41-hf" && xargs stat -c %s < /tmp/shards.txt | awk '{s+=$1} END {print s}') +have=$(cd "$LOCAL" && xargs stat -c %s < /tmp/shards.txt | awk '{s+=$1} END {print s}') +[[ $want == "$have" ]] +log "copy done: $(wc -l < /tmp/shards.txt) shards, $((have >> 30)) GiB" + +pip install -q --no-index --find-links "$W/wheels" -r "$KIT/requirements-ref.txt" \ + || pip install -q -r "$KIT/requirements-ref.txt" +nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$P/pod.log" + +log "router rows" +python3 "$KIT/keep_quant_torch.py" router --hf-path "$LOCAL" --out "$P/router-384.npz" | tee -a "$P/pod.log" + +log "smoke: two experts of layer 0" +rm -rf "$P/smoke" +python3 "$KIT/keep_quant_torch.py" fit --hf-path "$LOCAL" --importance "$P/imp-pruned.npz" --out "$P/smoke" \ + --layers 0 --experts 2 | tee -a "$P/pod.log" + +gpus=$(nvidia-smi -L | wc -l) +log "fit on $gpus GPU(s)" +for g in $(seq 0 $((gpus - 1))); do + CUDA_VISIBLE_DEVICES=$g python3 "$KIT/keep_quant_torch.py" fit --hf-path "$LOCAL" --importance "$P/imp-pruned.npz" \ + --out "$P/e8p-podimp-pruned" --layers "$(seq -s, "$g" "$gpus" 39)" > "$P/fit-$g.log" 2>&1 & +done +wait +[[ $(ls "$P/e8p-podimp-pruned"/layer-??.npz | wc -l) == 40 ]] +log "all done" +touch "$P/DONE" From 136cc1f31675ce4ae4cc2d62dde0456b3ffb99bc Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:40:07 -0800 Subject: [PATCH 56/74] v41_reap: extract_tensors.py, raw expert and router tensors from a pod as one safetensors stream --- scripts/runpod/v41_reap/extract_tensors.py | 76 +++++++++++++++++++++ scripts/runpod/v41_reap/keep_quant_torch.py | 2 +- 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 scripts/runpod/v41_reap/extract_tensors.py diff --git a/scripts/runpod/v41_reap/extract_tensors.py b/scripts/runpod/v41_reap/extract_tensors.py new file mode 100644 index 0000000..b0c6414 --- /dev/null +++ b/scripts/runpod/v41_reap/extract_tensors.py @@ -0,0 +1,76 @@ +"""Raw tensors out of an HF safetensors checkpoint, as one safetensors file on stdout. + +Either one layer's chosen routed experts, renumbered 0..n-1 in the order given, or every router +tensor of the decoder layers. Names are the reference names (``stream.reference_name``), bytes +are copied untouched. Standard library only, so it runs on any pod: + + ssh pod python3 extract_tensors.py /workspace/v41-hf --layer 3 --experts 1,2,5 > layer-03.safetensors + ssh pod python3 extract_tensors.py /workspace/v41-hf --gates > router.safetensors +""" + +import argparse +import json +import os +import re +import struct +import sys +from pathlib import Path + + +def reference_name(name: str) -> str: # stream.reference_name + name = name.removeprefix("model.").replace("self_attn", "attn") + if not name.startswith("vision."): + name = name.replace("mlp", "ffn") + return name.replace("weight_scale_inv", "scale").replace("e_score_correction_bias", "bias") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("root", type=Path) + p.add_argument("--layer", type=int) + p.add_argument("--experts", help="comma-separated expert ids of --layer") + p.add_argument("--gates", action="store_true") + args = p.parse_args() + index = json.loads((args.root / "model.safetensors.index.json").read_text())["weight_map"] + order = {int(e): i for i, e in enumerate(args.experts.split(","))} if args.experts else {} + picked: dict[str, tuple[str, str]] = {} # new name -> (shard, stored name) + for stored, shard in index.items(): + ref = reference_name(stored) + if args.gates: + if re.fullmatch(r"layers\.\d+\.ffn\.gate\..+", ref): + picked[ref] = (shard, stored) + else: + m = re.fullmatch(rf"layers\.{args.layer}\.ffn\.experts\.(\d+)\.(.+)", ref) + if m and int(m[1]) in order: + picked[f"layers.{args.layer}.ffn.experts.{order[int(m[1])]}.{m[2]}"] = (shard, stored) + assert picked and (args.gates or len(picked) == 6 * len(order)), len(picked) + + shards: dict[str, tuple[int, dict, int]] = {} + header, slabs, offset = {}, [], 0 + for name in sorted(picked): + shard, stored = picked[name] + if shard not in shards: + fd = os.open(args.root / shard, os.O_RDONLY) + (size,) = struct.unpack(" None: w = weights[proj][: args.rows] d = torch.from_numpy(full[space][layer][original].astype(np.float32)).to(device) started = time.time() - codes, scales, wrel = search.fit(w, d, 512 if w.shape[1] % 512 == 0 else 256) + codes, _, wrel = search.fit(w, d, 512 if w.shape[1] % 512 == 0 else 256) seconds = time.time() - started ref_codes, ref_scales = z[f"{proj}_codes"][expert, : args.rows], z[f"{proj}_scales"][expert, : args.rows] group = w.shape[1] // ref_scales.shape[1] From 0098ea8ddd8aa0dc6ee108f888ab6c259f99064f Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:56:49 -0800 Subject: [PATCH 57/74] v41_build: lattice cache over a pool wider than the loaded checkpoint (two code directories, full router file) --- scripts/v41_build/omlx_lattice.py | 60 +++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/scripts/v41_build/omlx_lattice.py b/scripts/v41_build/omlx_lattice.py index 2e7cb6c..9b9fd15 100644 --- a/scripts/v41_build/omlx_lattice.py +++ b/scripts/v41_build/omlx_lattice.py @@ -19,6 +19,14 @@ OMLX_COMPACT=DIR/compact-map.json OMLX_LATTICE=CODES_DIR python omlx_lattice.py serve ... python omlx_lattice.py check --config CONFIG.json --codes CODES_DIR +The pool can be wider than the loaded checkpoint. ``OMLX_LATTICE`` is then a JSON file:: + + {"codes": [[CODES_DIR, IDS.json], ...], "router": "router.safetensors", "mass": "reap-stats.npz"} + +``IDS.json`` (``{"survivors": {layer: [...]}}``) gives the pool number of each expert of its +directory, in file order; the first directory holds the loaded checkpoint's experts. ``router`` has +every router row of the pool and ``mass`` (``reap__gate_norm_sum``) orders the starting residents. + Environment: ``OMLX_LATTICE_SLOTS`` (resident experts per layer, default 144) and the policy settings of ``omlx_routed.py``. ``check`` compares one layer's decode step with dense matrices decoded on the host, the prefill path with the decode path, and slots after installs with their @@ -96,11 +104,12 @@ def _unused(dtype): class Codes: - """Where each expert's codes and scales sit inside the ``layer-NN.npz`` files.""" + """Where each expert's codes and scales sit inside the ``layer-NN.npz`` files of ``roots``. + ``where[layer][expert]`` is (directory, position in its file); without it, (0, expert).""" - def __init__(self, root: Path, layers: int): - self.files = [] - for i in range(layers): + def __init__(self, roots: list[Path], layers: int, where=None): + self.files, self.where = [[] for _ in range(layers)], where # [layer][directory] + for i, root in ((i, root) for i in range(layers) for root in roots): path = root / f"layer-{i:02d}.npz" fd, members = os.open(path, os.O_RDONLY), {} with zipfile.ZipFile(path) as z: @@ -114,16 +123,17 @@ def __init__(self, root: Path, layers: int): assert not fortran, info.filename start = info.header_offset + 30 + name + extra + f.tell() members[info.filename.removesuffix(".npy")] = (start, shape, dtype) - self.files.append((fd, members)) + self.files[i].append((fd, members)) def slab(self, layer: int, name: str, expert: int) -> tuple[int, int, int]: - fd, members = self.files[layer] + d, expert = self.where[layer][expert] if self.where else (0, expert) + fd, members = self.files[layer][d] start, shape, dtype = members[name] size = int(np.prod(shape[1:])) * dtype.itemsize return fd, start + expert * size, size def read(self, layer: int, name: str, experts, pool=None) -> np.ndarray: - _, shape, dtype = self.files[layer][1][name] + _, shape, dtype = self.files[layer][0][1][name] host = np.empty((len(experts), *shape[1:]), dtype) jobs = [(*self.slab(layer, name, e), host[s]) for s, e in enumerate(experts)] for done in [pool.submit(Store.pread_into, *job) for job in jobs] if pool else jobs: @@ -181,10 +191,10 @@ def views_current(self) -> bool: class LatticeCache(Cache): - def __init__(self, model, config, root: Path, order, full, slots: int): + def __init__(self, model, config, roots: list[Path], order, full, slots: int, where=None): super().__init__(config, mx) blocks = model.language_model.layers - self.codes = Codes(root, len(blocks)) + self.codes = Codes(roots, len(blocks), where) self.codebook = mx.array(e8p_packed_abs_grid()) layers = [] for i, blk in enumerate(blocks): @@ -292,6 +302,23 @@ def gate_plain(self, x, image_mask): # prefill and every unfused path language.Gate.__call__ = gate_plain +def wide_pool(path: Path, loaded: dict): + """(directories, order, router rows, where) of the pool file ``path``. ``loaded`` is the + router of the loaded checkpoint, whose rows must be the first directory's.""" + spec = json.loads(path.read_text()) + ids = [json.loads(Path(f).expanduser().read_text())["survivors"] for _, f in spec["codes"]] + where = [{e: (d, j) for d, per in enumerate(ids) for j, e in enumerate(per[str(i)])} for i in range(len(ids[0]))] + mass = np.load(Path(spec["mass"]).expanduser())["reap__gate_norm_sum"] + assert all(sorted(w) == list(range(mass.shape[1])) for w in where), "the directories do not cover the pool once" + router = mx.load(str(Path(spec["router"]).expanduser())) + full = {(i, f): router[f"layers.{i}.ffn.gate.{f}"] for i in range(len(where)) for f in ("weight", "bias")} + for (i, f), rows in loaded.items(): + if f in ("weight", "bias"): + assert mx.array_equal(full[i, f][mx.array(ids[0][str(i)])], rows), f"layer {i}: router {f} differs" + order = [np.argsort(-row, kind="stable").tolist() for row in mass] + return [Path(d).expanduser() for d, _ in spec["codes"]], order, full, where + + def patch(spec: dict, root: Path, slots: int) -> None: full, state = {}, {} compact_patch(spec, full) @@ -300,7 +327,11 @@ def patch(spec: dict, root: Path, slots: int) -> None: def load_lattice(*args, **kwargs): model, processor = load(*args, **kwargs) config = model.language_model.layers[0].ffn.gate._config - state["cache"] = LatticeCache(model, config, root, spec["order"], full, slots) + if root.is_file(): + roots, order, wide, where = wide_pool(root, full) + state["cache"] = LatticeCache(model, config, roots, order, wide, slots, where) + else: + state["cache"] = LatticeCache(model, config, [root], spec["order"], full, slots) full.clear() return model, processor @@ -327,7 +358,14 @@ def check(config_path: Path, root: Path, slots: int = 32, chain: int = 40) -> No full[i, "bias"] = mx.array(rng.normal(size=(pool,)).astype(np.float32) * 0.1) os.environ.setdefault("OMLX_ROUTED_TAU", "inf") model = SimpleNamespace(language_model=SimpleNamespace(layers=[SimpleNamespace(ffn=m) for m in moes])) - state = {"cache": LatticeCache(model, c, root, [rng.permutation(pool).tolist() for _ in moes], full, slots)} + roots, where = [root], None + if root.is_file(): # a pool file: both directories, the first two layers + roots, _, _, where = wide_pool(root, {}) + pool = len(where[0]) + for i in range(2): + full[i, "weight"] = mx.array(rng.normal(size=(pool, c.dim)).astype(np.float32)).astype(mx.bfloat16) + full[i, "bias"] = mx.array(rng.normal(size=(pool,)).astype(np.float32) * 0.1) + state = {"cache": LatticeCache(model, c, roots, [rng.permutation(pool).tolist() for _ in moes], full, slots, where)} cache = state["cache"] hook(state) From 12607cccd88832d35643e450657de1b4cec4df25 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:34:31 -0800 Subject: [PATCH 58/74] docs: the full pool of 384 serves on the M4 Max (144 lattice residents, 23-27 tok/s, 63 GiB) --- .../2026-11-08-v41-reap-recovery-log.md | 20 +++++++- .../2026-11-10-v41-wow-build-plan.md | 48 +++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 0a6d527..46e13e8 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -600,11 +600,11 @@ to 100; partial credit only for measured progress). | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | | 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 12 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. The rest needs the pool of 384 and the pod confirmation (E7) | | 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | -| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 7 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max with 144 2-bit lattice residents of the pool of 192 at 26–28 tok/s, 20 MB of real reads per token, 62 GiB (FP4, 64 residents: 22–26 tok/s). The rest needs the pool of 384 and memory pressure | +| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 8 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max over the full pool of 384 with 144 2-bit lattice residents at 23–27 tok/s (17–19 with thinking on), 80 MB of real reads per token, 63 GiB. The rest needs memory pressure | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **50** | | +| | **Total** | 100 | **51** | | ### 2026-11-09: prior art, plan, restart @@ -960,3 +960,19 @@ What it says: capacity. Nothing was created and nothing is billing beyond the volume. - Rubric unchanged at 50. +### 2026-11-10: pool of 384 on the Mac + +- No GPU was needed. A CPU pod (4 vCPU, $0.12/h, 76 minutes) streamed the 192 removed experts + per layer and all router rows to the Mac (144 GB); the Metal fit made their 2-bit codes in + 2.5 h (weighted error 0.091 on w1 and w3, 0.079 on w2). Pod terminated. +- The lattice cache now takes a pool wider than the loaded checkpoint (`0098ea8`). Over all 384 + experts with 144 residents: 26.6, 23.7 and 22.6 tok/s with thinking off, 19.1 and 17.3 with + thinking on, 63 GiB, 88–91% of demanded slots resident, 6–9 installs per token (1.0–2.1 GB/s + of reads). +- Text: one thinking-off script correct, one with a wrong token that crashes it; thinking on + finishes under sampling (2,704 reasoning tokens, correct script) and repeats under greedy + decoding like the pruned models. Table in the build plan, section "Pool of 384". +- Rubric item 6 moves from 7 to 8. Item 4 still needs the quality comparison over the pool of + 384, which needs the full FP4 model as reference (pod). Total 51. +- Disk: `pool384/fp4-removed` (144 GB of FP4 experts) is no longer needed by the cache and + waits for Jack's word before deletion. Nothing is running on the Mac or on Runpod. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 85fcc96..2b190e0 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -485,7 +485,49 @@ Text, same prompt as before (a script that finds duplicate files by SHA-256): - With the cache on, greedy output is not reproducible: install timing changes routing. Two runs diverged after 18 characters; a third matched the second. -Not done: the pool of 384 (codes exist only for the pruned 192), more than 144 residents (160 -would be about 67 GiB steady), memory pressure, and multi-token verify windows in serving (the -self-check covers 8 rows). +Not done: more than 144 residents (160 would be about 67 GiB steady), memory pressure, and +multi-token verify windows in serving (the self-check covers 8 rows). + +### Pool of 384 (2026-11-10) + +Every expert of the full checkpoint is now reachable on the Mac at 2 bits. + +- Codes. The 192 experts per layer that REAP-50 removed were copied from the Runpod volume + through a CPU pod (`extract_tensors.py`, 144 GB, 76 minutes, about $0.16) and fitted on the Mac + with the unchanged Metal search: 2.5 h for 40 layers, weighted error 0.091 on w1 and w3 and + 0.079 on w2 (the first 192: 0.090 and 0.074). The torch port refits 64 rows of nine + projections to the same codes. All 384 router rows per layer came with them; the rows of the + 192 survivors equal the REAP-50 checkpoint's. +- Cache. `OMLX_LATTICE` may name a pool file: code directories with the pool number of each + expert, a router file and routed-mass statistics for the starting residents. oMLX still loads + the REAP-50 checkpoint (everything outside the routed experts is shared), and the cache + replaces router and experts. Self-check over 384: same gaps as over 192; the decode-step MoE + takes 0.444 ms (0.40–0.42 over 192). +- Starting residents are the 144 experts with the most routed mass in calibration: 83% of the + mass on average, 66% in the worst layer. On average 118 of them are REAP-50 survivors. + +Serving, 144 residents, tau 0.2, 4 spares, 63 GiB (the pool of 192: 62 GiB): + +| Request | Tokens | tok/s | Demanded slots resident | Installs per token | Read | +|---|---:|---:|---:|---:|---:| +| Script, thinking off, greedy (first request) | 1,124 | 26.6 | 88% | 8.9 | 80 MB per token | +| Same again | 748 | 23.7 | 88% | 9.1 | 82 MB per token | +| 4,341-token prompt, summary | 282 | 22.6 | | | prefill 23.6 s (184 tok/s) | +| Script, thinking on, greedy, 4,096 budget | 4,096 | 19.1 | 91% | 6.3 | 1.05 GB/s | +| Script, thinking on, sampled, 12,000 budget | 3,256 | 17.3 | | 8.7 | 1.34 GB/s | + +- Against the pool of 192 at the same 144 residents: four times the installs (2.2 per token + there), 88–91% of demanded slots resident against 96%, and 0–15% lower speed with thinking + off (26.0–27.9 there). +- Text. The first thinking-off script runs correctly on the test tree. The second, from a + different cache state, has one wrong token (`sha.hexdigdigest()`) and crashes. One sample; + the pool of 192 produced no such error in its runs. +- Thinking on, greedy: no answer in 4,096 tokens, repeating from about character 12,800 of + 16,235, as the pruned models do. Thinking on, sampled: 2,704 reasoning tokens, then a + 552-token answer whose script runs correctly, 192 s in all (the pool of 192 needed 7,567 + reasoning tokens; one sample each). +- 7% of wanted admissions found no free slot in the first request, 5% over the thinking runs. + +Not measured: quality against the full FP4 model (it does not fit the Mac; pod work), other +values of tau, memory pressure. From 59c328284214237699101417be2c629fbf300bf1 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:17:40 -0800 Subject: [PATCH 59/74] v41_reap: local_eval.py on a GPU and on the unpruned checkpoint; tau384_pod.sh, the tau sweep over the pool of 384 --- scripts/runpod/v41_reap/keep_quant_torch.py | 7 +- scripts/runpod/v41_reap/local_eval.py | 96 ++++++++++++++------- scripts/runpod/v41_reap/tau384_pod.sh | 73 ++++++++++++++++ 3 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 scripts/runpod/v41_reap/tau384_pod.sh diff --git a/scripts/runpod/v41_reap/keep_quant_torch.py b/scripts/runpod/v41_reap/keep_quant_torch.py index aeea516..4c5de85 100644 --- a/scripts/runpod/v41_reap/keep_quant_torch.py +++ b/scripts/runpod/v41_reap/keep_quant_torch.py @@ -129,10 +129,11 @@ def load_expert(ckpt, layer: int, expert: int, device) -> dict[str, torch.Tensor def importance(args) -> None: from keep_quant import pod_importance - survivors = json.loads(args.survivors.read_text())["survivors"] + # Without --survivors: every expert, in order. + survivors = json.loads(args.survivors.read_text())["survivors"] if args.survivors else {} full = pod_importance(args.stats) # {space: [L, 384, in]} n_layers, n_experts = full["hidden"].shape[:2] - removed = np.array([sorted(set(range(n_experts)) - set(survivors[str(layer)])) for layer in range(n_layers)]) + removed = np.array([sorted(set(range(n_experts)) - set(survivors.get(str(layer), ()))) for layer in range(n_layers)]) out, replaced = {"experts": removed}, 0 for space, total in full.items(): picked = np.stack([total[layer][removed[layer]] for layer in range(n_layers)]) @@ -144,7 +145,7 @@ def importance(args) -> None: assert (picked.reshape(*picked.shape[:2], -1, 8) > 0).any(-1).all() and np.isfinite(picked).all(), space out[space] = picked.astype(np.float32) np.savez(args.out, **out) - print(f"{args.out}: {removed.shape[1]} removed experts in each of {n_layers} layers; " + print(f"{args.out}: {removed.shape[1]} experts in each of {n_layers} layers; " f"{replaced} (expert, space) importances replaced by the layer mean") diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 24901e8..5a99673 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -150,6 +150,28 @@ def mixed(layer: int, sd: dict) -> dict: return from_dir(Path(spec).expanduser()) +def layer_codes(spec: str, layer: int, have: list[int]) -> dict: + """One layer's code arrays, a row per expert of ``have`` (the checkpoint's experts, ascending). + + ``spec`` is a ``keep_quant.py`` directory holding exactly those experts, or + ``HOT_DIR@SET.json+REST_DIR``: the set's experts in one directory and the rest in the other, + each in ascending expert order.""" + if "@" not in spec: + return dict(np.load(Path(spec).expanduser() / f"layer-{layer:02d}.npz")) + hot_dir, _, rest = spec.partition("@") + set_path, _, rest_dir = rest.partition("+") + hot = set(json.loads(Path(set_path).expanduser().read_text())["survivors"][str(layer)]) + stored = [e for e in have if e in hot] + [e for e in have if e not in hot] # the two files back to back + row = {e: j for j, e in enumerate(stored)} + parts = [np.load(Path(d).expanduser() / f"layer-{layer:02d}.npz") for d in (hot_dir, rest_dir)] + assert sum(z["w1_codes"].shape[0] for z in parts) == len(have) and parts[0]["w1_codes"].shape[0] == len(hot & set(have)) + out = {k: np.concatenate([z[k] for z in parts])[[row[e] for e in have]] + for k in parts[0].files if k.endswith(("_codes", "_scales"))} + bits = {int(z["code_bits"]) if "code_bits" in z.files else 16 for z in parts} + assert len(bits) == 1, bits + return {**out, "code_bits": bits.pop()} + + def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: list[int], warm: np.ndarray | None, counts: np.ndarray, policy: str = "lru", trace: list | None = None, resident: np.ndarray | None = None) -> np.ndarray: @@ -205,7 +227,9 @@ def residual_cache(ind: np.ndarray, wts: np.ndarray, k: int, tau: float, bounds: def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--hf-path", required=True, type=Path) - p.add_argument("--checkpoint-survivors", required=True, type=Path, help="the survivors JSON the checkpoint was exported from") + p.add_argument("--checkpoint-survivors", type=Path, + help="the survivors JSON the checkpoint was exported from; omit for an unpruned checkpoint") + p.add_argument("--device", default="cpu", help="cpu, or cuda on a pod") p.add_argument("--pack", required=True, type=Path) p.add_argument("--stats", required=True, type=Path) p.add_argument("--sessions-file", required=True, type=Path) @@ -217,7 +241,9 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--expert-override", help="'fp4-bf16' or a keep_quant.py output directory") p.add_argument("--capture-imatrix", type=Path, help="measure the importance diagonal instead of scoring") p.add_argument("--dynamic", help="per-token tiers. BASE_DIR+RESIDUAL_DIR: base alone or base + nested residual. " - "BASE_DIR|HI_DIR: base alone or a separate high-precision copy (a hot expert then holds both)") + "BASE_DIR|HI_DIR: base alone or a separate high-precision copy (a hot expert then holds both). " + "With '|', a side may be HOT_DIR@SET.json+REST_DIR (the set's experts from one directory, the rest " + "from the other), and BASE may be empty when every variant has --miss") p.add_argument("--hi-proj", action="append", default=[], metavar="NAME=w2|w1w3", help="nested only, per variant: the high tier adds the residual for these projections only") p.add_argument("--cache-policy", action="append", default=[], metavar="NAME=lru|oracle", @@ -246,10 +272,14 @@ def main(argv: list[str] | None = None) -> int: import stream import torch_kernel - prune = json.loads((args.hf_path / "keep-prune.json").read_text()) - sha = hashlib.sha256(args.checkpoint_survivors.read_bytes()).hexdigest() - assert sha == prune["survivors_sha256"], "--checkpoint-survivors is not what this checkpoint was exported from" - ckpt_survivors = json.loads(args.checkpoint_survivors.read_text())["survivors"] + if args.checkpoint_survivors: + prune = json.loads((args.hf_path / "keep-prune.json").read_text()) + sha = hashlib.sha256(args.checkpoint_survivors.read_bytes()).hexdigest() + assert sha == prune["survivors_sha256"], "--checkpoint-survivors is not what this checkpoint was exported from" + ckpt_survivors = json.loads(args.checkpoint_survivors.read_text())["survivors"] + else: # an unpruned checkpoint: every expert, under its own number + shape = json.loads((args.hf_path / "inference" / "config.json").read_text()) + ckpt_survivors = {str(l): list(range(shape["n_routed_experts"])) for l in range(shape["n_layers"])} variants = {} for spec in args.variant: name, _, path = spec.partition("=") @@ -259,8 +289,13 @@ def main(argv: list[str] | None = None) -> int: torch_kernel.exact_quant = True torch.set_num_threads(args.threads) ref_dir = args.hf_path / "inference" - ref = stream.import_reference(ref_dir, kernel=torch_kernel) + dev = args.device + # On a GPU the reference's own kernels run, as in collect.py; elsewhere the torch stand-ins. + ref = stream.import_reference(ref_dir, kernel=torch_kernel if dev == "cpu" else None) torch.set_default_dtype(torch.bfloat16) + if dev != "cpu": + torch.cuda.memory._set_allocator_settings("expandable_segments:True") + torch.set_default_device(dev) torch.manual_seed(33377335) model_args = ref.ModelArgs(**json.loads((ref_dir / "config.json").read_text())) model_args.max_batch_size = 1 @@ -280,9 +315,10 @@ def main(argv: list[str] | None = None) -> int: ckpt = stream.HFCheckpoint(args.hf_path) layout = ref.EngramLayout.from_args(model_args) - hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) + with torch.device(dev): + hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) engram_rows = {l: stream.EngramRows.from_checkpoint(ckpt, l, ref.fp8_block_size) for l in model_args.engram_layer_ids} - streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, "cpu") + streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, dev) state = collect.install_hooks(streamer, top_k=2048, rows_per_slice=256) n_local = len(next(iter(ckpt_survivors.values()))) @@ -294,8 +330,7 @@ def main(argv: list[str] | None = None) -> int: copy_mode = "|" in args.dynamic # the high tier is its own copy, not base + residual args_cache_k_names = {s.partition("=")[0] for s in args.cache_k} - base_dir, _, res_dir = args.dynamic.partition("|" if copy_mode else "+") - base_dir, res_dir = Path(base_dir).expanduser(), Path(res_dir).expanduser() + tier_specs = [s for s in args.dynamic.split("|" if copy_mode else "+", 1) if s] # base, then high hi_proj = {name: {"w2": ("w2",), "w1w3": ("w1", "w3")}[v] for name, _, v in (s.partition("=") for s in args.hi_proj)} assert not (copy_mode and hi_proj), "--hi-proj needs nested codes" policies = {name: v for name, _, v in (s.partition("=") for s in args.cache_policy)} @@ -306,7 +341,8 @@ def main(argv: list[str] | None = None) -> int: assert all(m in ("drop", "reroute") and copy_mode and v in args_cache_k_names for v, m in miss.items()), \ "--miss takes drop or reroute, and needs BASE|HI codes and --cache-k for the same variant" assert set(taus) == set(variants), "--tau needs one NAME=VALUE per --variant" - np_grids = {16: e8p_full_grid().astype(np.float32), 8: e8_1bit_grid().astype(np.float32)} + assert len(tier_specs) == 2 or set(miss) == set(variants), "codes without a base need --miss for every variant" + grids = {bits: torch.as_tensor(grid().astype(np.float32), device=dev) for bits, grid in ((16, e8p_full_grid), (8, e8_1bit_grid))} cache: dict = {"layer": None} n_layers_ = int(model_args.n_layers) cache_k = {name: int(k) for name, _, k in (s.partition("=") for s in args.cache_k)} @@ -329,20 +365,20 @@ def tier_weights(layer: int, e: int) -> tuple[dict, dict]: if cache["layer"] != layer: cache.clear() cache["layer"] = layer - cache["z"] = [dict(np.load(d / f"layer-{layer:02d}.npz")) for d in (base_dir, res_dir)] + cache["z"] = [layer_codes(s, layer, sorted(ckpt_survivors[str(layer)])) for s in tier_specs] if e not in cache: tiers = [] for z in cache["z"]: - grid = np_grids[int(z.get("code_bits", 16))] # the first E8P runs wrote no code_bits + grid = grids[int(z.get("code_bits", 16))] # the first E8P runs wrote no code_bits tier = {} for proj in ("w1", "w2", "w3"): - vec = grid[z[f"{proj}_codes"][e].astype(np.int64)] - s = z[f"{proj}_scales"][e].astype(np.float32) - vec = vec * np.repeat(s, vec.shape[1] // s.shape[-1], axis=-1)[..., None] - tier[proj] = vec.reshape(vec.shape[0], -1) + vec = grid[torch.as_tensor(z[f"{proj}_codes"][e].astype(np.int64), device=dev)] # [out, in/8, 8] + s = torch.as_tensor(z[f"{proj}_scales"][e].astype(np.float32), device=dev) + tier[proj] = (vec * s.repeat_interleave(vec.shape[1] // s.shape[-1], dim=-1)[..., None]).flatten(-2) tiers.append(tier) - lo = {p: torch.as_tensor(w).bfloat16() for p, w in tiers[0].items()} - hi = {p: torch.as_tensor(tiers[1][p] if copy_mode else tiers[0][p] + tiers[1][p]).bfloat16() for p in tiers[0]} + lo = {p: w.bfloat16() for p, w in tiers[0].items()} + hi = lo if len(tiers) == 1 else {p: (tiers[1][p] if copy_mode else lo_w + tiers[1][p]).bfloat16() + for p, lo_w in tiers[0].items()} cache[e] = (lo, hi) return cache[e] @@ -363,9 +399,10 @@ def dynamic_forward(self, x, image_mask=None): on_miss = miss.get(state["variant"]) resident = np.zeros((x.size(0), self.gate.weight.size(0)), dtype=bool) if on_miss == "reroute" else None hi_all = None if k is None else torch.as_tensor( - simulate_cache(layer, indices.numpy(), weights.float().numpy(), k, tau, counts, resident)) + simulate_cache(layer, indices.cpu().numpy(), weights.float().cpu().numpy(), k, tau, counts, resident), + device=x.device) if resident is not None: # demand (above) drove the cache; the tokens run on what is resident - weights, indices = gate_masked(self.gate, x, torch.as_tensor(resident), ref.linear) + weights, indices = gate_masked(self.gate, x, torch.as_tensor(resident, device=x.device), ref.linear) hi_all = weights > 0 for i in torch.unique(indices).tolist(): idx, top = torch.where(indices == i) @@ -439,11 +476,11 @@ def build_with_override(layer, sd): if args.capture_imatrix: assert len(variants) == 1, "--capture-imatrix takes one variant" capture = collect.SessionStats(torch=torch, n_layers=int(model_args.n_layers), n_experts=n_local, - dims={"hidden": int(model_args.dim), "down": int(model_args.moe_inter_dim)}, device="cpu") + dims={"hidden": int(model_args.dim), "down": int(model_args.moe_inter_dim)}, device=dev) state["bounds"] = [len(r["token_ids"]) for r in records.values()] # one MoE call holds a variant's records in order items = { - (v, n): stream.Item(tokens=torch.as_tensor([r["token_ids"]], dtype=torch.long), group=(v, None)) + (v, n): stream.Item(tokens=torch.as_tensor([r["token_ids"]], dtype=torch.long, device=dev), group=(v, None)) for v in variants for n, r in records.items() } @@ -455,7 +492,7 @@ def before_moe(layer: int, block, key) -> None: bias.data.copy_(block.full_bias) masks = variants[key[0]] if masks is not None: - bias.data[masks[layer]] = float("-inf") + bias.data[masks[layer].to(bias.device)] = float("-inf") state["stats"] = capture state["variant"] = key[0] @@ -486,11 +523,12 @@ def before_moe(layer: int, block, key) -> None: assert np.array_equal(positions, teacher["positions"][rows]), name targets = np.asarray(record["token_ids"], dtype=np.int64)[positions + 1] teacher_nll = (teacher["logsumexp"][rows] - teacher["target_logit"][rows]).astype(np.float64) - state["positions"] = torch.as_tensor(positions) - state["targets"] = torch.as_tensor(targets) + state["positions"] = torch.as_tensor(positions, device=dev) + state["targets"] = torch.as_tensor(targets, device=dev) state["teacher"] = { - "ids": torch.as_tensor(teacher["topk_logit_ids"][rows].astype(np.int64)), - "logprob": torch.as_tensor(teacher["topk_logit_values"][rows].astype(np.float32) - teacher["logsumexp"][rows, None]), + "ids": torch.as_tensor(teacher["topk_logit_ids"][rows].astype(np.int64), device=dev), + "logprob": torch.as_tensor(teacher["topk_logit_values"][rows].astype(np.float32) - teacher["logsumexp"][rows, None], + device=dev), "targets": state["targets"], } for v in variants: diff --git a/scripts/runpod/v41_reap/tau384_pod.sh b/scripts/runpod/v41_reap/tau384_pod.sh new file mode 100644 index 0000000..d8f1aab --- /dev/null +++ b/scripts/runpod/v41_reap/tau384_pod.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Tau sweep over the pool of 384 on one GPU pod. +# +# 1. Fit 2-bit E8P codes for all 384 experts of every layer (keep_quant_torch.py, the fit the Mac +# serves; uploading the Mac's 130 GB of codes would take 12 h at its 3 MB/s uplink). +# 2. Simulate the expert cache that omlx_lattice.py serves (local_eval.py: 144 residents, re-route +# on a miss) at several admission thresholds, scored against the teacher's logits on the Mac's +# 22 eval sessions (first 1,024 tokens each). +# +# On the volume before the GPU starts (a CPU pod put it there): +# /workspace/v41-hf, /workspace/pack, /workspace/wheels, /workspace/keep +# /workspace/pool384/{imp-all.npz,reap-stats.npz,sessions.json} +# /workspace/pool384/stats/eval/sessions/NAME.npz teacher logits +# Output on the volume: pool384/codes/e8p-384 (the codes), pool384/tau-sweep/summary.json, DONE. +set -euo pipefail +W=/workspace +P=$W/pool384 +KIT=$W/keep/scripts/runpod/v41_reap +LOCAL=${LOCAL:-/root/v41} +CODES=$LOCAL/codes/e8p-384 +export PYTHONPATH=$W/keep/src PIP_BREAK_SYSTEM_PACKAGES=1 +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$P/pod.log"; } +trap 'status=$?; [[ $status -ne 0 ]] && { log "FAILED (status $status)"; touch "$P/FAILED"; }' EXIT + +# Never run from the volume (one stream is 120 MB/s): the checkpoint goes to the container's +# disk with 16 streams (about 480 GB). +log "copy checkpoint to $LOCAL/hf" +mkdir -p "$LOCAL/hf" +(cd "$W/v41-hf" && find . -type f -not -path './.cache/*' -print0 | xargs -0 -P 16 -I{} cp --parents {} "$LOCAL/hf/") +[[ $(cd "$W/v41-hf" && find . -type f -not -path './.cache/*' -printf '%s\n' | awk '{s+=$1} END {print s}') == $(cd "$LOCAL/hf" && find . -type f -printf '%s\n' | awk '{s+=$1} END {print s}') ]] +log "copy done: $(du -sh "$LOCAL/hf" | cut -f1)" + +pip install -q --no-index --find-links "$W/wheels" -r "$KIT/requirements-ref.txt" \ + || pip install -q -r "$KIT/requirements-ref.txt" +nvidia-smi --query-gpu=name,memory.total --format=csv | tee -a "$P/pod.log" + +fit() { python3 "$KIT/keep_quant_torch.py" fit --hf-path "$LOCAL/hf" --importance "$P/imp-all.npz" "$@"; } +log "fit smoke: four experts of layer 0" +rm -rf "$LOCAL/fit-smoke" +fit --out "$LOCAL/fit-smoke" --layers 0 --experts 4 | tee -a "$P/pod.log" +hours=$(python3 -c "import json,sys; print(round(json.loads(open(sys.argv[1]).readline())['seconds'] / 4 * 384 * 40 / 3600, 2))" "$LOCAL/fit-smoke/metrics.jsonl") +log "fit of all 15,360 experts projected at $hours h" +python3 -c "import sys; sys.exit(float(sys.argv[1]) > float(sys.argv[2]))" "$hours" "${MAX_FIT_H:-4}" || { log "over MAX_FIT_H, stopping"; exit 1; } +fit --out "$CODES" > "$P/fit.log" 2>&1 +[[ $(find "$CODES" -name 'layer-??.npz' | wc -l) == 40 ]] +log "fit done: $(tail -1 "$P/fit.log")" +mkdir -p "$P/codes/e8p-384" # keep the codes for later pod runs; the copy runs beside the sweep +(cd "$CODES" && find . -type f -print0 | xargs -0 -P 8 -I{} cp {} "$P/codes/e8p-384/") & +keep=$! + +run() { + python3 "$KIT/local_eval.py" --device cuda --hf-path "$LOCAL/hf" --pack "$W/pack/v41-pack.jsonl" --stats "$P/stats" \ + --max-rss-gb 150 --cache-warm "$P/reap-stats.npz" --dynamic "|$CODES" "$@" +} +v() { echo --variant "$1=all" --tau "$1=$3" --cache-k "$1=$2" --miss "$1=reroute" ${4:+--cache-policy "$1=$4"}; } + +log "sweep smoke: one session, 128 tokens" +echo '["eval-00000"]' > "$P/smoke-sessions.json" +# shellcheck disable=SC2046 +run --sessions-file "$P/smoke-sessions.json" --max-tokens 128 $(v q384 384 0) $(v t20 144 0.2) --out "$P/smoke-out" 2>&1 | tail -5 | tee -a "$P/pod.log" + +# q384: every expert resident (2-bit coding alone). s144: the 144 hottest, never changed. +# tNN: the cache of 144 at tau 0.NN. o144: each record's own best 144 (the policy's ceiling). +log "sweep: 22 sessions, 1,024 tokens, 7 variants" +# shellcheck disable=SC2046 +run --sessions-file "$P/sessions.json" --max-tokens 1024 \ + $(v q384 384 0) $(v s144 144 1e9) $(v t10 144 0.1) $(v t20 144 0.2) $(v t30 144 0.3) $(v t45 144 0.45) $(v o144 144 0.2 oracle) \ + --out "$P/tau-sweep" > "$P/sweep.log" 2>&1 +python3 "$KIT/routed_report.py" --eval "$P/tau-sweep" | tee -a "$P/pod.log" +wait $keep +[[ $(find "$P/codes/e8p-384" -name 'layer-??.npz' | wc -l) == 40 ]] +log "all done" +touch "$P/DONE" From 46fff8129e7c7318b661577bc96d7cee47523812 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:43:56 -0800 Subject: [PATCH 60/74] v41_reap: local_eval.py --router, the pool of 384 scored on a pruned checkpoint from codes alone --- scripts/runpod/v41_reap/local_eval.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 5a99673..1be1bcd 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -230,6 +230,8 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--checkpoint-survivors", type=Path, help="the survivors JSON the checkpoint was exported from; omit for an unpruned checkpoint") p.add_argument("--device", default="cpu", help="cpu, or cuda on a pod") + p.add_argument("--router", type=Path, help="safetensors with every router row of the unpruned model: the pool is " + "all its experts, taken from --dynamic codes alone (HOT_DIR@SET.json+REST_DIR), on a pruned checkpoint") p.add_argument("--pack", required=True, type=Path) p.add_argument("--stats", required=True, type=Path) p.add_argument("--sessions-file", required=True, type=Path) @@ -280,6 +282,14 @@ def main(argv: list[str] | None = None) -> int: else: # an unpruned checkpoint: every expert, under its own number shape = json.loads((args.hf_path / "inference" / "config.json").read_text()) ckpt_survivors = {str(l): list(range(shape["n_routed_experts"])) for l in range(shape["n_layers"])} + wide = None + if args.router: # the checkpoint's own experts are never run: every variant re-routes over the codes + from safetensors.torch import load_file + + assert args.dynamic and {v.partition("=")[0] for v in args.miss} == {v.partition("=")[0] for v in args.variant}, \ + "--router needs --dynamic codes and --miss for every variant" + wide, loaded = load_file(args.router), ckpt_survivors + ckpt_survivors = {l: list(range(wide[f"layers.{l}.ffn.gate.bias"].numel())) for l in loaded} variants = {} for spec in args.variant: name, _, path = spec.partition("=") @@ -319,6 +329,20 @@ def main(argv: list[str] | None = None) -> int: hash_state = ref.NgramHashState(model_args, layout, AutoTokenizer.from_pretrained(args.hf_path)) engram_rows = {l: stream.EngramRows.from_checkpoint(ckpt, l, ref.fp8_block_size) for l in model_args.engram_layer_ids} streamer = stream.LayerStreamer(ref, model_args, ckpt.load, engram_rows, hash_state, dev) + if wide: + build_narrow = streamer.build_block + + def build_wide(layer, sd): + block = build_narrow(layer, sd) + gate, ids = block.ffn.gate, loaded[str(layer)] + for f in ("weight", "bias", "bias_vl"): + if getattr(gate, f, None) is not None: + rows = wide[f"layers.{layer}.ffn.gate.{f}"].to(getattr(gate, f)) + assert torch.equal(rows[ids], getattr(gate, f).data), f"layer {layer}: router {f} is not this checkpoint's" + setattr(gate, f, torch.nn.Parameter(rows, requires_grad=False)) + return block + + streamer.build_block = build_wide state = collect.install_hooks(streamer, top_k=2048, rows_per_slice=256) n_local = len(next(iter(ckpt_survivors.values()))) From c5ec2d45c77bd1a4a51e98f6588e3cd28f3704cd Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:11:20 -0800 Subject: [PATCH 61/74] docs: tau sweep over the pool of 384 (cache of 144 at KL 0.454 against 0.763 static; tau 0.3 as default) --- .../2026-11-08-v41-reap-recovery-log.md | 22 ++++++++- .../2026-11-10-v41-wow-build-plan.md | 48 ++++++++++++++++++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 46e13e8..de93a71 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -598,13 +598,13 @@ to 100; partial credit only for measured progress). | 1 | Nested codes, all layers, validated | 10 | 10 | fit 40/40; error 0.102 gate/up, 0.084 down; end to end +0.053 KL against flat 2-bit | | 2 | Per-token routed tiers, implemented and tested | 10 | 10 | `--dynamic/--tau/--tau-drop`, smoke-tested | | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | -| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 12 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. The rest needs the pool of 384 and the pod confirmation (E7) | +| 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 20 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. Pool of 384 (2026-11-10): a cache of 144 beats the static 144 by 0.309 (21 of 22 sessions) and the exact REAP-50 checkpoint by 0.12, at 0.7 GB/s for tau 0.3. The rest needs the pod confirmation (E7) | | 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | | 6 | Real Apple Silicon streaming measured, not simulated | 10 | 8 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max over the full pool of 384 with 144 2-bit lattice residents at 23–27 tok/s (17–19 with thinking on), 80 MB of real reads per token, 63 GiB. The rest needs memory pressure | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **51** | | +| | **Total** | 100 | **59** | | ### 2026-11-09: prior art, plan, restart @@ -976,3 +976,21 @@ What it says: 384, which needs the full FP4 model as reference (pod). Total 51. - Disk: `pool384/fp4-removed` (144 GB of FP4 experts) is no longer needed by the cache and waits for Jack's word before deletion. Nothing is running on the Mac or on Runpod. + +### 2026-11-10: tau sweep over the pool of 384, on the Mac + +- Jack asked for the sweep on a pod; US-NE-1 (the volume's data center) had no GPU, and he + chose the Mac. `local_eval.py --router` (`46fff81`) scores the full pool on the REAP-50 + checkpoint from the 2-bit codes alone, against the full model's stored logits. +- KL over 22 sessions, 1,024 tokens: all 384 resident 0.390; cache of 144 at tau 0.1, 0.2, 0.3, + 0.45: 0.483, 0.460, 0.454, 0.573; oracle 144 0.424; static 144 0.763. The exact REAP-50 + checkpoint scores 0.578 on the same tokens. +- The cache beats the static set by 0.309 (standard error 0.052, 21 of 22 sessions). Tau 0.3 + equals 0.2 at a third of the reads (2.6 installs per token, 0.7 GB/s at 30 tok/s). +- Serving at tau 0.3: 27.2, 26.7 and 23.0 tok/s, 1.4–2.5 installs per token, both scripts + correct. Tau 0.3 is the serving default. Tables in the build plan, section "Tau sweep over + the pool of 384". +- Rubric item 4 moves from 12 to 20 of 25; the remaining 5 wait for the pod confirmation (E7). + Total 59. +- Run: 11,991 s, peak RSS 63.8 GB. The pod job `tau384_pod.sh` was not needed. Nothing is + running on the Mac or on Runpod. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 2b190e0..341ef50 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -528,6 +528,50 @@ Serving, 144 residents, tau 0.2, 4 spares, 63 GiB (the pool of 192: 62 GiB): reasoning tokens; one sample each). - 7% of wanted admissions found no free slot in the first request, 5% over the thinking runs. -Not measured: quality against the full FP4 model (it does not fit the Mac; pod work), other -values of tau, memory pressure. +Not measured: memory pressure. + +### Tau sweep over the pool of 384 (2026-11-10) + +Quality against the full FP4 model needs only its stored logits, not the model: the pod wrote +them for the Mac's 22 eval sessions. `local_eval.py --router` puts the 384-row router on the +REAP-50 checkpoint and takes every routed expert from the 2-bit codes the Mac serves, so the +simulated cache (144 residents, re-route on a miss, one-token lag, starting residents by routed +mass) runs on the Mac. First 1,024 tokens of each session, KL against the full model: + +| Variant | KL | Top-1 | Installs per token | Read at 30 tok/s | +|---|---:|---:|---:|---:| +| All 384 resident (2-bit coding alone) | 0.390 | 84.4% | 0 | 0 | +| Cache of 144, best set per record (oracle) | 0.424 | 84.0% | | | +| Cache of 144, tau 0.1 | 0.483 | 82.5% | 17.1 | 4.6 GB/s | +| Cache of 144, tau 0.2 | 0.460 | 82.7% | 7.6 | 2.1 GB/s | +| Cache of 144, tau 0.3 | 0.454 | 82.5% | 2.6 | 0.7 GB/s | +| Cache of 144, tau 0.45 | 0.573 | 80.6% | 1.1 | 0.3 GB/s | +| Static 144 (hottest by routed mass) | 0.763 | 76.2% | 0 | 0 | + +For reference, same sessions and tokens: the exact REAP-50 checkpoint (192 FP4 experts) 0.578, +REAP-50 at 2 bits 0.747, the cache of 144 inside the pool of 192 0.756. + +- The cache of 144 over the full pool beats the static 144 by 0.309 (standard error 0.052, 21 + of 22 sessions) and the exact REAP-50 checkpoint by 0.12, at 53 GB of resident experts. +- Tau 0.2 and 0.3 are equal (difference 0.005, standard error 0.017) and 0.3 reads a third as + much. Tau 0.1 is worse than 0.2 (0.024, standard error 0.013): admitting weak slots evicts + useful experts. Tau 0.45 loses 0.12 in every session. +- At tau 0.3 the cache is 0.030 from the oracle and 0.064 from having every expert resident. +- Tool-use text stays at 1.8–2.0 in every variant, all 384 resident included (1.94), so that + loss is the 2-bit coding and not the cache. +- Run: 11,991 s, peak RSS 63.8 GB (`~/Models/v41-reap-run/jobs/tau384_mac.sh`). + +Serving at tau 0.3 (144 residents, 4 spares, 63 GiB), against the tau 0.2 table above: + +| Request | Tokens | tok/s | Demanded slots resident | Installs per token | Read | +|---|---:|---:|---:|---:|---:| +| Script, thinking off, greedy (first request) | 1,030 | 27.2 | 80% | 2.5 | 22 MB per token | +| Same again | 1,085 | 26.7 | 83% | 1.4 | 13 MB per token | +| 4,341-token prompt, summary | 310 | 23.0 | 71% | | prefill 24.8 s (175 tok/s) | + +Both scripts run correctly on the test tree. Installs match the simulation (2.6 per token). +Tau 0.3 is the serving default from here. + +Limits: the cache is simulated with a fetch that lands within one token; 22 sessions, first +1,024 tokens; two generated scripts at tau 0.3; no pod confirmation on longer sessions (E7). From d6cf35d33dd505474fdcb52fa89a893f810e47fd Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:14:58 -0800 Subject: [PATCH 62/74] v41_build: k5_pagecache.py, the page-cache kill test (MLX aliases a mapping; Metal pages in the whole buffer) --- scripts/v41_build/k5_pagecache.py | 122 ++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 scripts/v41_build/k5_pagecache.py diff --git a/scripts/v41_build/k5_pagecache.py b/scripts/v41_build/k5_pagecache.py new file mode 100644 index 0000000..e2ee61a --- /dev/null +++ b/scripts/v41_build/k5_pagecache.py @@ -0,0 +1,122 @@ +"""K5: can macOS's page cache hold expert bytes that MLX computes on in place? + +The idea under test: map one file holding every expert, alias it as an MLX array, and let the +kernel decide which pages stay in memory. That only works if + +1. MLX aliases a file mapping (no copy into its own buffers), +2. using a small part of the aliased buffer on the GPU leaves the rest out of memory. + +Result on the M4 Max, mlx 0.32.4 (2026-11-11): 1 holds for every mapping mode (same address, no +page touched by aliasing). 2 fails: a kernel that reads 1 MiB of a 2 GiB aliased buffer brings +every page of it into memory. One buffer over all experts would therefore page in the whole +pool on first use, so the design is dead in this form; the cost of a cold read was not measured +because no read is cold after the first. + +Each trial writes its own file with F_NOCACHE, so its pages start out of the page cache +(checked with mincore). Prints one JSON line per measurement. + + ~/Development/_refs/omlx/.venv/bin/python scripts/v41_build/k5_pagecache.py DIR [--gb 2] +""" + +import argparse +import ctypes +import fcntl +import json +import mmap +import os +import time +from pathlib import Path + +import mlx.core as mx +import numpy as np + +PAGE = mmap.PAGESIZE +libc = ctypes.CDLL(None, use_errno=True) + + +def cold_file(path: Path, nbytes: int) -> None: + """``nbytes`` of uint16 noise, written past the page cache.""" + chunk = np.random.default_rng(0).integers(0, 1 << 16, 1 << 23, dtype=np.uint16).tobytes() + assert nbytes % len(chunk) == 0 and len(chunk) % PAGE == 0 + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + fcntl.fcntl(fd, fcntl.F_NOCACHE, 1) + for _ in range(nbytes // len(chunk)): + assert os.write(fd, chunk) == len(chunk) + os.fsync(fd) + os.close(fd) + + +def in_memory(m: np.memmap) -> np.ndarray: + """One flag per page of the mapping: is it in memory (mincore)?""" + vec = (ctypes.c_char * (m.nbytes // PAGE))() + if libc.mincore(ctypes.c_void_p(m.ctypes.data), ctypes.c_size_t(m.nbytes), vec): + raise OSError(ctypes.get_errno(), "mincore") + return (np.frombuffer(vec, np.uint8) & 1).astype(bool) + + +def alias(m: np.memmap) -> mx.array: + a = mx.asarray(m, copy=False) # raises if MLX would copy + mx.eval(a) + return a + + +def timed_sum(a: mx.array, start: int, count: int) -> float: + t = time.perf_counter() + mx.eval(a[start : start + count].astype(mx.float32).sum()) + return (time.perf_counter() - t) * 1e3 + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("dir", type=Path) + p.add_argument("--gb", type=int, default=2) + args = p.parse_args() + args.dir.mkdir(parents=True, exist_ok=True) + nbytes, mb = args.gb << 30, (1 << 20) // 2 # mb: uint16 elements per MiB + say = lambda **kw: print(json.dumps(kw), flush=True) + say(mlx=mx.__version__, page=PAGE, file_gb=args.gb, wired_limit=mx.set_wired_limit(0)) + mx.eval(mx.zeros(8).sum()) # Metal is up before anything is timed + + # 1. Aliasing, for each way of mapping the file. + modes = {} + for mode in ("r", "r+", "c"): + path = args.dir / f"alias-{mode.replace('+', 'w')}.bin" + cold_file(path, nbytes) + m = np.memmap(path, dtype=np.uint16, mode=mode) + cold = float(in_memory(m).mean()) + active = mx.get_active_memory() + try: + a = alias(m) + except Exception as e: # noqa: BLE001 (the message is the result) + say(test="alias", mode=mode, ok=False, error=f"{type(e).__name__}: {e}"[:200]) + continue + same = np.asarray(memoryview(a)).ctypes.data == m.ctypes.data + say(test="alias", mode=mode, ok=True, same_address=bool(same), cold_before=cold, + in_memory_after_alias=float(in_memory(m).mean()), mlx_active_added_mb=(mx.get_active_memory() - active) >> 20) + if same: + modes[mode] = path + del a, m + if not modes: + say(verdict="dead: MLX copies every mapping from Python; try allocator::make_buffer natively") + return + mode = next(iter(modes)) + + # 2. Does touching 1 MiB on the GPU bring the whole buffer into memory? + for where in (0, nbytes // 4): # element offsets: the start and the middle of the file + path = args.dir / f"touch-{where}.bin" # a new file each time: a rewritten one keeps its cached pages + cold_file(path, nbytes) + m = np.memmap(path, dtype=np.uint16, mode=mode) + a = alias(m) + before = in_memory(m) + ms = timed_sum(a, where, mb) + after = in_memory(m) + say(test="touch-1MiB", at_gb=where * 2 / (1 << 30), ms=round(ms, 2), in_memory_before=float(before.mean()), + in_memory_after=float(after.mean()), pages_after=int(after.sum()), pages_needed=(mb * 2) // PAGE) + del a, m + + for f in args.dir.glob("*.bin"): + f.unlink() + + +if __name__ == "__main__": + main() From 76384aa8331f500e90672180b0c59c5ebf8bef1c Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:49:36 -0800 Subject: [PATCH 63/74] v41_reap: local_eval.py --hot, the hottest residents as the checkpoint's FP4 experts; K5 result in the build plan --- .../2026-11-10-v41-wow-build-plan.md | 2 +- scripts/runpod/v41_reap/local_eval.py | 36 ++++++++++++++++--- scripts/runpod/v41_reap/routed_report.py | 5 +++ tests/test_runpod_v41_routed_precision.py | 7 ++++ 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 341ef50..3b260cf 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -139,7 +139,7 @@ Red-team of the chosen plan: | K2 | `bench_e8p_m1` at 5120 × 2304 against oMLX's 2-bit affine kernel. **Passed 2026-11-10:** per routed expert the lattice kernel takes 0.037 ms, oMLX's affine 2-bit 0.047 and its FP4 0.042; the lattice expert is 9.0 MB against 11.1. | Our lattice kernel in the runtime: ship affine 2-bit instead. | | K3 | Static 250 and 288 of 384 against the full model, teacher-forced, on the pod. **Done 2026-11-10:** KL 0.323 at 250 and 0.235 at 288 (0.532 at 192, 1.123 at 96), native precision, no cache. | Nothing; it sets how much the cache has to recover. At the 80 GB cap (about 160 experts) a fixed set costs roughly 0.6–0.7 by interpolation, so the cap matters more than the code density. | | K4 | E1: expert cache with re-route against static sets, teacher-forced. **Passed 2026-11-10:** the cache beats the static set by 0.056 KL at 144 residents and 0.231 at 96 (pool of 192, 2 bits); a cache of 96 equals a static 144; the oracle gap (0.020) is all in the first 256 tokens. | The quality case for the cache, if it is not better than a static set. | -| K5 | 20 lines: an MLX array over a 1 GB memory-mapped file through the E8P kernel; check footprint and `mincore`. | Idea 3 (page cache), if it copies. | +| K5 | 20 lines: an MLX array over a 1 GB memory-mapped file through the E8P kernel; check footprint and `mincore`. **Failed 2026-11-11** (`scripts/v41_build/k5_pagecache.py`, mlx 0.32.4): `mx.asarray(np.memmap, copy=False)` does alias the mapping (same address, no page touched), but a GPU kernel that reads 1 MiB of a 2 GiB aliased buffer brings all 131,072 pages into memory (0.79 s). One buffer over all experts would page in the whole pool. | Idea 3 (page cache), if it copies. Dead in the one-buffer form; one buffer per expert would need the routed experts known on the host before dispatch, which costs a sync per layer. | | K6 | Offline, from stored routes: how well token ids predict experts; how well a prompt's demand predicts the rest of the session; recall of the drafter's 256-d head. | Ideas 2, 6 and 7 respectively. | ## 4. Critical path diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 1be1bcd..3aa7b5e 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -90,21 +90,30 @@ def gate_masked(gate, x, allowed, linear): return weights * gate.route_scale, indices +def decode_fp4(weight, scale): + """bf16 [out, in] of one FP4 projection (E2M1 codes, one scale per 32 inputs).""" + import torch_kernel + + return (torch_kernel.decode_e2m1(weight).unflatten(-1, (-1, 32)) * scale.float()[..., None]).flatten(-2).bfloat16() + + +def hot_set(order, n: int, have) -> set[int]: + """The first ``n`` experts of ``order`` (hottest first) that are among ``have``.""" + have = set(have) + return set([int(e) for e in order if int(e) in have][:n]) + + def expert_override(spec: str, n_experts: int, ckpt_survivors: dict): """``layer -> {(expert, proj): bf16 [out, in]}`` from FP4 (``fp4-bf16``) or E8P codes.""" import torch - import torch_kernel - if spec == "fp4-bf16": def from_state(layer: int, sd: dict) -> dict: out = {} for e in range(n_experts): for proj in ("w1", "w2", "w3"): - w = torch_kernel.decode_e2m1(sd[f"ffn.experts.{e}.{proj}.weight"]) - s = sd[f"ffn.experts.{e}.{proj}.scale"].float() - out[(e, proj)] = (w.unflatten(-1, (-1, 32)) * s[..., None]).flatten(-2).bfloat16() + out[(e, proj)] = decode_fp4(sd[f"ffn.experts.{e}.{proj}.weight"], sd[f"ffn.experts.{e}.{proj}.scale"]) return out return from_state @@ -261,6 +270,9 @@ def main(argv: list[str] | None = None) -> int: "skipped (drop), or the router picks its top-k among resident experts (reroute)") p.add_argument("--cache-k", action="append", default=[], metavar="NAME=K", help="with --dynamic: hi tier only from an LRU of K resident residuals per layer, admitted at gate >= tau") + p.add_argument("--hot", action="append", default=[], metavar="NAME=H", + help="with --miss: the H hottest experts (--cache-warm order) that the checkpoint holds run from " + "its FP4 weights while resident, the other residents from the codes") p.add_argument("--cache-warm", type=Path, help="imatrix npz (reap__gate_norm_sum): warm-start each cache with the top-K") p.add_argument("--sidecar", type=Path, help="a sidecar_fit.py output directory, added to every variant's MoE output") p.add_argument("--sidecar-rank", type=int, help="use only the top-k sidecar components (stored largest first)") @@ -373,6 +385,10 @@ def build_wide(layer, sd): warm = None if args.cache_warm: # warm start: each layer's top-K experts by routed mass (the static hot set) warm = np.argsort(-np.load(args.cache_warm)["reap__gate_norm_sum"], axis=1, kind="stable") + hot_n = {name: int(h) for name, _, h in (s.partition("=") for s in args.hot)} + assert set(hot_n) <= set(miss) and (warm is not None or not hot_n), "--hot needs --miss and --cache-warm" + own = loaded if wide else ckpt_survivors # pool numbers of the checkpoint's own experts, in its order + hot_slots = dict.fromkeys(variants, 0) for v in variants: # [layer, expert, (lo slots, hi slots, lo mass, hi mass, admissions, dropped slots, dropped mass)] dyn_counts[v] = np.zeros((n_layers_, n_local, 7)) @@ -439,6 +455,15 @@ def dynamic_forward(self, x, image_mask=None): counts[i, 6] += float(w[drop].sum()) hi_mask = hi_mask & ~drop lo_w, hi_w = tier_weights(layer, i) + if hot_n.get(state["variant"]): # residents of the hot set run the checkpoint's FP4 expert + if "hot" not in cache: + cache["hot"] = {n: hot_set(warm[layer], n, own[str(layer)]) for n in set(hot_n.values())} + if i in cache["hot"][hot_n[state["variant"]]]: + if ("fp4", i) not in cache: + mine = self.experts[own[str(layer)].index(i)] + cache["fp4", i] = {p: decode_fp4(getattr(mine, p).weight, getattr(mine, p).scale) for p in lo_w} + hi_w = cache["fp4", i] + hot_slots[state["variant"]] += int(hi_mask.sum()) only = hi_proj.get(state["variant"]) if only is not None: # a finer rung: the residual of some projections only hi_w = {p: hi_w[p] if p in only else lo_w[p] for p in lo_w} @@ -596,6 +621,7 @@ def before_moe(layer: int, block, key) -> None: "policy": policies.get(v, "lru") if v in cache_k else None, "copy_mode": copy_mode, "hi_units": hi_units, "n_bases": 0 if v in miss else kept, "pool": kept, "miss": miss.get(v), + "hot": hot_n.get(v, 0), "hot_slot_share": hot_slots[v] / max(tot[:2].sum() + tot[5], 1), "tau_drop": tau_drop.get(v, 0.0), "hi_slot_share": float(tot[1] / (tot[:2].sum() + tot[5])), "drop_slot_share": float(tot[5] / (tot[:2].sum() + tot[5])), diff --git a/scripts/runpod/v41_reap/routed_report.py b/scripts/runpod/v41_reap/routed_report.py index ca8dc27..3802b7b 100644 --- a/scripts/runpod/v41_reap/routed_report.py +++ b/scripts/runpod/v41_reap/routed_report.py @@ -24,6 +24,7 @@ DIM, INTER = 5120, 2304 # 1-bit E8 code + fp16 group scales: gate/up group 512 (1.031 bpw), down group 256 (1.0625 bpw) UNIT_BYTES = (2 * DIM * INTER * (1 + 16 / 512) + DIM * INTER * (1 + 16 / 256)) / 8 # ~4.6 MB +FP4_BYTES = 3 * DIM * INTER * (4 + 8 / 32) / 8 # an FP4 expert with one scale byte per 32 weights, 18.8 MB def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: @@ -59,6 +60,10 @@ def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: "ssd_mb_per_tok": round(ssd / 1e6, 1), "ssd_gbs": round(ssd * tok_s / 1e9, 2), "read_gb_per_tok": round(read / 1e9, 2), "tok_s_bound": round(bandwidth_gbs * 1e9 / read, 1), } + if tiers.get("hot"): # a cache whose hottest experts are FP4 copies + mixed = (k - tiers["hot"]) * hi_units * UNIT_BYTES + tiers["hot"] * FP4_BYTES + row |= {"hot": tiers["hot"], "hot_slots": round(tiers["hot_slot_share"], 3), + "resident_gb": round(mixed * n_layers / 1e9, 1)} out.append(row) return out diff --git a/tests/test_runpod_v41_routed_precision.py b/tests/test_runpod_v41_routed_precision.py index d14905a..21a5147 100644 --- a/tests/test_runpod_v41_routed_precision.py +++ b/tests/test_runpod_v41_routed_precision.py @@ -126,3 +126,10 @@ def test_gate_masked_routes_among_allowed_experts(): few[:, :2] = True # fewer allowed than top-k: the surplus slot carries no weight w3, i3 = local_eval.gate_masked(gate, x, few, torch.nn.functional.linear) assert ((w3 > 0).sum(-1) == 2).all() and (i3[w3 > 0] < 2).all() + + + +def test_hot_set_takes_the_hottest_experts_the_checkpoint_holds(): + assert local_eval.hot_set(np.array([7, 2, 9, 4, 1]), 2, [1, 2, 4]) == {2, 4} + assert local_eval.hot_set(np.array([7, 2]), 3, [2, 7, 8]) == {2, 7} + assert routed_report.FP4_BYTES == 18_800_640 From cca338af61e472dd9fbaf9b64fc84d6f6ff25761 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:55:17 -0800 Subject: [PATCH 64/74] docs: drafter on the no-wait cache (+26-43% tok/s), sampling at the card's settings, tool-use KL note corrected --- .../2026-11-08-v41-reap-recovery-log.md | 22 ++++++++ .../2026-11-10-v41-wow-build-plan.md | 54 ++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index de93a71..65b9dd4 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -994,3 +994,25 @@ What it says: Total 59. - Run: 11,991 s, peak RSS 63.8 GB. The pod job `tau384_pod.sh` was not needed. Nothing is running on the Mac or on Runpod. + +### 2026-11-11: plan for four ideas; page cache dead, drafter works, sampling check + +- Jack picked four ideas (drafter on the no-wait cache, higher precision on the hottest experts, + one download for any Mac, the OS page cache as the cache) and approved the plan. He also + pointed at the upstream model card: generation tests now use its settings (temperature 1.0, + top_p 1.0, large budgets) and say so when they depart. +- Page cache (K5, `scripts/v41_build/k5_pagecache.py`, `d6cf35d`): MLX 0.32.4 aliases a file + mapping, but a kernel reading 1 MiB of a 2 GiB aliased buffer brings every page into memory. + Dead in the one-buffer form. +- Drafter: on through `mtp_enabled` with no code change; +26–28% on scripts, +43% with thinking + on, +9% on a long prompt, +7 GiB, at 120 residents with installs byte-checked. Above the 1.25 + gate on three of four requests. A multi-row kernel would add at most about 12% and is parked. +- Sampling at the card's settings: 4 to 7 of 8 sampled scripts run correctly across the cache + (120 and 144 residents, drafter on and off, temperature 1.0 and 0.6) and the exact REAP-50 + checkpoint (5 of 8). Single-token slips; no row separable at 8 samples; greedy scripts all + correct. More samples queued. +- `local_eval.py --hot` (`76384aa`) runs the hottest residents from the checkpoint's FP4 + weights. Smoke (1 session, 96 tokens): cache of 144 1.52, with 32 hot 1.14; all 384 resident + with 32 hot 0.68. The resident-count curve (C1) and the hot-tier comparison (B1) are running. +- Rubric unchanged at 59: no item's own test was met. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 3b260cf..bed4fbc 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -557,8 +557,10 @@ REAP-50 at 2 bits 0.747, the cache of 144 inside the pool of 192 0.756. much. Tau 0.1 is worse than 0.2 (0.024, standard error 0.013): admitting weak slots evicts useful experts. Tau 0.45 loses 0.12 in every session. - At tau 0.3 the cache is 0.030 from the oracle and 0.064 from having every expert resident. -- Tool-use text stays at 1.8–2.0 in every variant, all 384 resident included (1.94), so that - loss is the 2-bit coding and not the cache. +- Tool-use text stays at 1.8–2.0 in every variant, all 384 resident included (1.94). That is + the off-format teacher found on 2026-11-09 (the eval text carries foreign tool markup), not a + loss from the cache or the coding. Without that domain: 0.23 with all 384 resident, 0.32 for + the cache of 144 at tau 0.3, 0.66 for the static 144. - Run: 11,991 s, peak RSS 63.8 GB (`~/Models/v41-reap-run/jobs/tau384_mac.sh`). Serving at tau 0.3 (144 residents, 4 spares, 63 GiB), against the tau 0.2 table above: @@ -575,3 +577,51 @@ Tau 0.3 is the serving default from here. Limits: the cache is simulated with a fetch that lands within one token; 22 sessions, first 1,024 tokens; two generated scripts at tau 0.3; no pod confirmation on longer sessions (E7). +### Drafter on the no-wait cache (2026-11-11) + +The checkpoint's drafter (DSpark: 3 stages, 128 experts each, 7.4 GiB) is switched on by +`mtp_enabled` in `/model_settings.json`; no code change was needed. A cycle drafts 3 +tokens and verifies them in one backbone forward of about 4 rows through the same hook the +single-token path uses. 120 residents (so the footprint stays under the 80 GB cap), tau 0.3, +pool of 384, installed slots byte-checked (0 mismatches): + +| Request | Drafter off | Drafter on | Gain | +|---|---:|---:|---:| +| Script, thinking off, greedy | 24.3 tok/s | 31.2 | +28% | +| Same, temperature 1.0, top_p 1.0 | 24.2 | 30.5 | +26% | +| 4,341-token prompt, summary | 21.8 | 23.9 | +9% | +| Script, thinking on, temperature 1.0 | 17.45 (3,872 tokens) | 25.0 (8,492 tokens) | +43% | +| Footprint, steady / peak | 55 / 61 GiB | 62 / 68 GiB | +7 GiB | + +- About 2.9 tokens are kept per cycle; a cycle takes 93 ms against 41 ms for a single token. +- Installs per forward row are unchanged; per emitted token they are 1.38 times the no-drafter + figure (0.67 GB/s read against 0.41 with thinking on). +- oMLX's own offload gains 29% from the drafter (6.6 to 8.5 tok/s) because verification waits + on experts; here nothing waits. +- A multi-row lattice kernel (one decode per distinct expert in the block) is not worth + building yet: over the full model's routes, distinct experts per route are 0.84 at 2 rows, + 0.70 at 4 and 0.63 at 6, so it saves at most about 12% of a cycle. +- Job: `~/Models/v41-reap-run/jobs/k1_mtp.sh`. + +### Sampling at the card's settings (2026-11-11) + +The same script request, 8 samples per row, 4,096-token budget, thinking off. "Correct" means +the script runs and lists the three duplicate files of the test tree. + +| Model | Residents | Temperature | Drafter | Correct of 8 | +|---|---:|---:|---|---:| +| Cache over the pool of 384, 2-bit | 120 | 1.0 | off | 4 | +| Same | 120 | 1.0 | on | 4 | +| Same | 144 | 1.0 | off | 6 | +| Same | 120 | 0.6 | off | 7 | +| Exact REAP-50 checkpoint (192 FP4 experts, stock oMLX offload, 4–5 tok/s) | 192 | 1.0 | off | 5 | + +- Failures are single-token slips (`hexdigist`, `mun` for `min`, a constant misspelled at its + use, a missing colon) and two samples with wrong output. Every greedy script was correct. +- With 8 samples no row is separable from another (4 of 8 against 7 of 8 is p = 0.28, Fisher). + The exact pruned checkpoint at its original precision fails 3 of 8, so the slips are not + evidence against the 2-bit coding, the cache or the drafter. +- Not known: how often the full model slips at temperature 1.0 (it does not fit on the Mac), + and whether more samples separate the rows. 16 more per row for the cache of 144 and the + exact checkpoint are queued behind the simulator run. +- Jobs: `~/Models/v41-reap-run/jobs/k1_mtp_samples.sh`; scorer in the session scratchpad. From 650fa6a72b04b7b86fa20fb0aeb5d331acbfe512 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:47:33 -0800 Subject: [PATCH 65/74] docs: KL by resident count over the pool of 384 (cache of 48 level with static 144), FP4 on the hottest experts (gate not met below 200 residents) --- .../2026-11-08-v41-reap-recovery-log.md | 18 +++++ .../2026-11-10-v41-wow-build-plan.md | 68 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 65b9dd4..417e43c 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1016,3 +1016,21 @@ What it says: with 32 hot 0.68. The resident-count curve (C1) and the hot-tier comparison (B1) are running. - Rubric unchanged at 59: no item's own test was met. +### 2026-11-11: quality against resident count (C1) and FP4 on the hottest experts (B1) + +- C1, pool of 384, tau 0.3, 22 sessions, KL (without tool-use in brackets): 32 residents 0.942 + (0.732), 48 0.790 (0.582), 64 0.705 (0.496), 96 0.605 (0.390), 144 0.454 (0.320), 200 0.401 + (0.279), 250 0.398 (0.254), all 384 0.390 (0.235). A cache of 48 (28 GiB footprint) is level + with the static 144 (60 GiB); a cache of 96 (44 GiB) is level with the exact REAP-50 + checkpoint. Breadth is spent near 200 residents. +- B1, gate of 0.02 over the equal-memory 2-bit rival: 16 FP4 experts at 53 GB −0.041 (−0.010 + without tool-use); 144 with 32 FP4 against 177 at 65 GB −0.029 (−0.015). Met only with the + tool-use sessions counted, which carry the off-format teacher, so not met on the reading to + trust. Trading 34 residents for 32 FP4 experts loses 0.022. +- Precision alone: all 384 resident with 32 FP4 per layer 0.268 against 0.390 (0.167 against + 0.235 without tool-use, all 20 sessions). 144 with 32 FP4 at 65 GB equals 200 at 74 GB. +- Reading: below about 200 residents breadth buys more than bits; above it only bits are left. + The 128 GB budget (92 GB of experts) is the untested case; `c1b1_mac.sh b1b` (217 with 32 + FP4, 183 with 64 FP4, against 250 at 2 bits) is queued behind the serving runs. +- Tables in the build plan, sections C1 and B1. Rubric unchanged at 59. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index bed4fbc..84928bf 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -625,3 +625,71 @@ the script runs and lists the three duplicate files of the test tree. and whether more samples separate the rows. 16 more per row for the cache of 144 and the exact checkpoint are queued behind the simulator run. - Jobs: `~/Models/v41-reap-run/jobs/k1_mtp_samples.sh`; scorer in the session scratchpad. + +### Quality against resident count (C1, 2026-11-11) + +Same simulator, sessions and tokens as the tau sweep: pool of 384, tau 0.3, re-route on a miss, +22 sessions, first 1,024 tokens, KL against the full FP4 model. Footprint is the measured +serving model (11.8 GiB outside routed experts plus 0.336 GiB per resident slot). + +| Residents per layer | Resident experts | Footprint | KL | KL without tool-use | Top-1 | Read per token | +|---:|---:|---:|---:|---:|---:|---:| +| 32 | 11.8 GB | 23 GiB | 0.942 | 0.732 | 73.5% | 118 MB | +| 48 | 17.7 GB | 28 GiB | 0.790 | 0.582 | 77.1% | 82 MB | +| 64 | 23.6 GB | 33 GiB | 0.705 | 0.496 | 78.7% | 61 MB | +| 96 | 35.4 GB | 44 GiB | 0.605 | 0.390 | 81.1% | 39 MB | +| 144 | 53.1 GB | 60 GiB | 0.454 | 0.320 | 82.5% | 24 MB | +| 200 | 73.7 GB | 79 GiB | 0.401 | 0.279 | 83.8% | 13 MB | +| 250 | 92.2 GB | 96 GiB | 0.398 | 0.254 | 84.4% | 5.5 MB | +| 384 (all) | 141.6 GB | | 0.390 | 0.235 | 84.4% | 0 | +| Static 144, for reference | 53.1 GB | 60 GiB | 0.763 | 0.654 | 76.2% | 0 | + +- 144 reproduces the tau sweep (0.4542, every session equal). +- A cache of 48 at 28 GiB is level with the static 144 at 60 GiB (0.790 against 0.763; 0.582 + against 0.654 without tool-use), and a cache of 96 at 44 GiB is level with the exact REAP-50 + checkpoint (0.605 against 0.578), which needs 341 GB of FP4 experts on disk. +- 144 to 200 gains 0.053 (standard error 0.009, 22 of 22 sessions). 200 to 250 gains 0.003 + over all sessions and 0.024 without tool-use: breadth is spent near 200 residents. +- Reads grow as the cache shrinks: 2.5 GB/s at 30 tok/s for 48 residents, 3.5 GB/s for 32. +- Run: 12,400 s (`~/Models/v41-reap-run/jobs/c1b1_mac.sh`). + +### FP4 on the hottest experts (B1, 2026-11-11) + +`local_eval.py --hot NAME=H`: the H hottest experts per layer by routed mass, among those the +REAP-50 checkpoint holds, run from its FP4 weights whenever they are resident (18.8 MB against +9.2 MB at 2 bits). This is the ceiling for any extra bits on those experts. + +| Variant | Resident experts | KL | Without tool-use | Top-1 | +|---|---:|---:|---:|---:| +| 144 at 2 bits | 53.1 GB | 0.454 | 0.320 | 82.5% | +| 127 residents, 16 of them FP4 | 53.0 GB | 0.413 | 0.310 | 83.3% | +| 110 residents, 32 of them FP4 | 52.8 GB | 0.476 | 0.331 | 82.9% | +| 177 at 2 bits | 65.2 GB | 0.421 | 0.294 | 83.2% | +| 144 residents, 32 of them FP4 | 65.4 GB | 0.392 | 0.280 | 83.9% | +| 200 at 2 bits | 73.7 GB | 0.401 | 0.279 | 83.8% | +| All 384 at 2 bits | 141.6 GB | 0.390 | 0.235 | 84.4% | +| All 384, 32 of them FP4 | 153.8 GB | 0.268 | 0.167 | 86.9% | + +Paired by session (difference, standard error, sessions better): + +| Comparison | All 22 sessions | 20 without tool-use | +|---|---|---| +| 16 FP4 against 144 at 2 bits, 53 GB | −0.041 ± 0.023 (12) | −0.010 ± 0.006 (10) | +| 32 FP4 against 144 at 2 bits, 53 GB | +0.022 ± 0.010 (5) | +0.011 ± 0.006 (5) | +| 144 with 32 FP4 against 177 at 2 bits, 65 GB | −0.029 ± 0.013 (14) | −0.015 ± 0.009 (12) | +| 144 with 32 FP4 against 144 at 2 bits (12 GB more) | −0.062 ± 0.016 (19) | −0.041 ± 0.008 (17) | +| All 384 with 32 FP4 against all 384 at 2 bits | −0.122 ± 0.039 (22) | −0.067 ± 0.013 (20) | + +- Gate (0.02 over the equal-memory 2-bit rival): met with tool-use counted (0.041 and 0.029), + not met without it (0.010 and 0.015). The tool-use sessions carry the off-format teacher, so + the reading without them is the one to trust. At 53 and 65 GB, FP4 on the hot set is a + small gain at best, and trading 34 residents for 32 FP4 experts is a loss. +- Precision alone is large: with every expert resident, FP4 on 32 experts per layer (8% of + them) removes 0.067 of the 0.235 left, in all 20 sessions. Those experts take 22% of routed + slots. +- 144 residents with 32 at FP4 (65 GB) equals 200 residents at 2 bits (74 GB). +- So extra bits pay where breadth is spent, above about 200 residents, which is the budget of + a 128 GB Mac. That budget was not tested here; 217 residents with 32 FP4 and 183 with 64 + FP4 against 250 at 2 bits (all 92 GB) is the deciding run. +- Run: 8,549 s, peak RSS 61 GB. + From ccca2bc68ff53192b37425c202e75ff978fd0fca Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:21:35 -0800 Subject: [PATCH 66/74] docs: serving at 48 and 96 residents (speed holds, sampled scripts fail: 0 and 2 of 8) --- .../2026-11-08-v41-reap-recovery-log.md | 11 ++++++++++ .../2026-11-10-v41-wow-build-plan.md | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 417e43c..639c4da 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1034,3 +1034,14 @@ What it says: FP4, 183 with 64 FP4, against 250 at 2 bits) is queued behind the serving runs. - Tables in the build plan, sections C1 and B1. Rubric unchanged at 59. +### 2026-11-11: serving at 48 and 96 residents (C2) + +- Card sampling, 8 sampled scripts each: 48 residents (30 GiB, 20–27 tok/s) 0 correct, 96 + residents (46 GiB, 22–25 tok/s) 2 correct. With 120 and 144 from earlier today: 0, 2, 4, 6 of + 8. Text at 48 residents is misspelled throughout and most samples never finish. +- KL understates this: the simulator put a cache of 48 level with the static 144 (0.79 against + 0.76), and that level is unusable at temperature 1.0. "One download, any Mac" does not hold + at the card's sampling below about 144 residents on this evidence. +- Queued: temperature 0.6 at 96 and 48 residents, top_p 0.95 at 144 and 96; before those, 16 + more samples for the cache of 144 and the exact checkpoint, and the 92 GB hot-tier run. + diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 84928bf..0128b6a 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -693,3 +693,25 @@ Paired by session (difference, standard error, sessions better): FP4 against 250 at 2 bits (all 92 GB) is the deciding run. - Run: 8,549 s, peak RSS 61 GB. +### Serving at 48 and 96 residents (C2, 2026-11-11) + +Same sampled script request as above, card sampling (temperature 1.0, top_p 1.0), 8 samples, +drafter off, tau 0.3. Fewer slots stand in for a smaller Mac's expert memory, not for its +memory pressure. + +| Residents | Footprint | tok/s | Demanded slots resident | Installs per token | Read per token | Correct of 8 | +|---:|---:|---:|---:|---:|---:|---:| +| 48 | 30 GiB | 20–27 | 65% | 5.8 | 53 MB | 0 | +| 96 | 46 GiB | 22–25 | 75% | 3.2 | 28 MB | 2 | +| 120 (above) | 55 GiB | 24 | | | | 4 | +| 144 (above) | 63 GiB | 17–27 | | | | 6 | + +- Speed holds at small sizes; the text does not. At 48 residents the output is misspelled + throughout (`hashllib`, `OSEError`, `o a file`) and 6 of 8 samples run to the 4,096-token + limit. At 96, 3 of 8 run to the limit. +- Correct scripts rise with residents: 0, 2, 4, 6 of 8 at 48, 96, 120, 144. A KL near 0.8 is + unusable under the card's sampling and 0.45 is marginal, so the KL curve above overstates + what a small Mac gets. The exact REAP-50 checkpoint (KL 0.578) gives 5 of 8. +- Open: whether a lower temperature or a trimmed tail (top_p 0.95) makes 96 residents usable, + and how many of 8 the full model gets. The first is queued (`jobs/after_b1b.sh`). + From 7d18b31730ee2f6fd14c6abd860dd462375b0841 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:17:40 -0800 Subject: [PATCH 67/74] docs: hot tier at 92 GB (217 residents with 32 FP4 beat 250 at 2 bits by 0.072), 24-sample script check, temperature and top_p rescue checks --- .../2026-11-08-v41-reap-recovery-log.md | 18 ++++++ .../2026-11-10-v41-wow-build-plan.md | 64 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 639c4da..b8de326 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1045,3 +1045,21 @@ What it says: - Queued: temperature 0.6 at 96 and 48 residents, top_p 0.95 at 144 and 96; before those, 16 more samples for the cache of 144 and the exact checkpoint, and the 92 GB hot-tier run. + +### 2026-11-11: hot tier at 92 GB (B1b), more samples, rescue checks + +- B1b, the expert budget of a 128 GB Mac: 217 residents with the hottest 32 at FP4 score 0.325 + (0.219 without tool-use) against 0.398 (0.254) for 250 at 2 bits, both 92 GB. Paired: −0.072 + ± 0.027 (17 of 22), −0.036 ± 0.010 without tool-use (15 of 20). The 0.02 gate is met on both + readings, so B continues. 183 with 64 FP4 scores the same (0.328, 0.221). +- 217 with 32 FP4 at 92 GB is at least level with all 384 resident at 2 bits at 139 GB (−0.064 + ± 0.035; −0.016 ± 0.009 without tool-use). Bits lose to breadth at 53 GB, draw at 65 GB and + win at 92 GB. +- Sampled scripts at the card's settings, 24 each: cache of 144 at 2 bits 17 correct, exact + REAP-50 18 correct; the cache runs at 22–27 tok/s against 5.6–6.3. +- Rescue checks, 8 each: 96 residents at temperature 0.6 1 correct, 48 residents 0; top_p 0.95 + gives 5 at 144 and 2 at 96. Neither setting rescues a small cache; the errors are misspelled + identifiers from re-routed slots. The usable floor on this request is about 120 residents. +- Serving the 92 GB configuration needs about 96 GiB of footprint, above the 80 GB rule. +- Tables in the build plan, sections B1b and "More samples". Rubric unchanged at 59: no item's + own test was met. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 0128b6a..f3b71d5 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -715,3 +715,67 @@ memory pressure. - Open: whether a lower temperature or a trimmed tail (top_p 0.95) makes 96 residents usable, and how many of 8 the full model gets. The first is queued (`jobs/after_b1b.sh`). + +### FP4 on the hottest experts at 92 GB (B1b, 2026-11-11) + +The deciding run named above: the expert budget of a 128 GB Mac (92 GB), pool of 384, tau 0.3, +re-route, 22 sessions, 1,024 tokens. KL against the full model; "no tool-use" drops the two +off-format sessions. + +| Variant | Residents | Expert memory | KL | KL, no tool-use | Top-1 | +|---|---|---:|---:|---:|---:| +| 250 at 2 bits (C1) | 250 | 92 GB | 0.398 | 0.254 | 84.4% | +| 217, hottest 32 at FP4 | 217 | 92.3 GB | 0.325 | 0.219 | 85.6% | +| 183, hottest 64 at FP4 | 183 | 92.0 GB | 0.328 | 0.221 | 86.1% | +| All 384 at 2 bits (C1) | 384 | 139 GB | 0.390 | 0.235 | | +| All 384, hottest 32 at FP4 (B1) | 384 | 154 GB | 0.268 | 0.167 | 86.9% | + +Paired differences, mean ± standard error (sessions better): + +| Pair | All 22 | Without tool-use (20) | +|---|---|---| +| 217 with 32 FP4 against 250 at 2 bits | −0.072 ± 0.027 (17) | −0.036 ± 0.010 (15) | +| 183 with 64 FP4 against 250 at 2 bits | −0.070 ± 0.030 (16) | −0.034 ± 0.011 (14) | +| 217 with 32 FP4 against 200 at 2 bits (74 GB) | −0.075 ± 0.016 (21) | −0.060 ± 0.009 (19) | +| 217 with 32 FP4 against all 384 at 2 bits (139 GB) | −0.064 ± 0.035 (15) | −0.016 ± 0.009 (13) | +| 183 with 64 FP4 against 217 with 32 FP4 | +0.002 ± 0.015 | +0.002 ± 0.003 | + +- Gate (0.02 over the equal-memory 2-bit rival): met on both readings, 0.072 with tool-use and + 0.036 without. B continues. +- At 92 GB, 217 residents with 32 at FP4 are at least level with every expert resident at 2 + bits, which takes 139 GB. The gain is in the agentic, networking and cyber sessions (0.32 to + 0.25, 0.25 to 0.16, 0.48 to 0.38); prose and business sessions do not move. +- 32 and 64 FP4 experts per layer score the same, so the hot set is small: 32 is enough, and + the choice between them can be made on serving cost. +- With B1: bits lose to breadth at 53 GB, draw at 65 GB, win at 92 GB. The crossover is near + 200 residents. +- Serving this needs about 96 GiB of footprint (185 slots at 2 bits, 32 at FP4, 11.8 GiB + outside the routed experts), above the 80 GB per-process rule on this Mac. Under the rule + the hot tier can be served at about 130 residents with 32 at FP4. +- Run: 3,339 s, peak RSS 60 GB. + +### More samples and two rescue checks (2026-11-11) + +Same sampled script request, 8 samples per run, 4,096-token budget, drafter off. + +| Model | Residents | Temperature | top_p | Correct | tok/s | +|---|---:|---:|---:|---|---:| +| Cache, pool of 384 at 2 bits | 144 | 1.0 | 1.0 | 17 of 24 (6, 5, 6) | 22–27 | +| Exact REAP-50, FP4, stock offload | 192 | 1.0 | 1.0 | 18 of 24 (5, 6, 7) | 5.6–6.3 | +| Cache | 144 | 1.0 | 0.95 | 5 of 8 | 24–27 | +| Cache | 120 | 0.6 | 1.0 | 7 of 8 (earlier) | | +| Cache | 96 | 1.0 | 1.0 | 2 of 8 (C2) | 22–25 | +| Cache | 96 | 1.0 | 0.95 | 2 of 8 | 24–26 | +| Cache | 96 | 0.6 | 1.0 | 1 of 8 | 20–28 | +| Cache | 48 | 0.6 | 1.0 | 0 of 8 | 23–25 | + +- With 24 samples each, the cache of 144 at 2 bits and the exact REAP-50 checkpoint are level + (71% and 75%) at the card's sampling, and the cache is about four times as fast. Their + failures are of the same kind: one misspelled name or a wrong output. +- Neither a lower temperature nor top_p 0.95 rescues 96 or 48 residents. The failures at 96 + with temperature 0.6 are still misspelled identifiers (`hexdigdigest`, `diggest`, + `filnames`), so the damage comes from the re-routed slots (25% of demand at 96 residents, + 35% at 48), not from the sampling tail. top_p 0.95 does nothing at 144 either. +- Between 96 and 120 residents at temperature 0.6 the count goes from 1 to 7 of 8. The usable + floor on this request is about 120 residents (55 GiB), which is a 96 GB Mac, not a 64 GB one. +- Still unknown: how many of 24 the full model gets (it does not fit on this Mac). From 0ab2081c6db7d4df21689a7904a028d4e95370dd Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:30:40 -0800 Subject: [PATCH 68/74] docs: 200 residents served under the 100 GB limit (19 of 24 sampled scripts, no spelling failures, 87 GiB peak) --- .../2026-11-08-v41-reap-recovery-log.md | 8 ++++++++ .../2026-11-10-v41-wow-build-plan.md | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index b8de326..9121eb3 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1063,3 +1063,11 @@ What it says: - Serving the 92 GB configuration needs about 96 GiB of footprint, above the 80 GB rule. - Tables in the build plan, sections B1b and "More samples". Rubric unchanged at 59: no item's own test was met. + +### 2026-11-11: 200 residents served + +- Per-process limit raised to 100 GB for serving (Jack). The cache runs at 200 residents: 81 + GiB at start, 87 GiB peak, 90% of demanded slots resident, 0.40 installs per token. +- Sampled scripts at the card's settings: 19 of 24 correct (144 residents 17, exact REAP-50 + 18). No failure is a misspelled name or syntax error; at 144 residents most were. +- Speed not measured cleanly (the residual fit shared the GPU); queued. Rubric unchanged at 59. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index f3b71d5..c409eb4 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -779,3 +779,23 @@ Same sampled script request, 8 samples per run, 4,096-token budget, drafter off. - Between 96 and 120 residents at temperature 0.6 the count goes from 1 to 7 of 8. The usable floor on this request is about 120 residents (55 GiB), which is a 96 GB Mac, not a 64 GB one. - Still unknown: how many of 24 the full model gets (it does not fit on this Mac). + +### Serving 200 residents (2026-11-11) + +Jack raised the per-process limit to 100 GB for serving. Same sampled script request, card +sampling, 24 samples, drafter off, tau 0.3, 200 residents at 2 bits over the pool of 384. + +| Residents | Footprint, peak | Demanded slots resident | Installs per token | Read per token | Correct of 24 | +|---:|---:|---:|---:|---:|---:| +| 144 (above) | 63 GiB | | | | 17 | +| 200 | 87 GiB | 90% | 0.40 | 3.6 MB | 19 | +| Exact REAP-50 (above) | | | | | 18 | + +- 19 of 24, and none of the 5 failures is a misspelled name or a syntax error: 4 scripts run + and print the wrong list, 1 answer has no code block. At 144 residents 5 of 7 failures were + misspelled names or syntax errors. Fewer re-routed slots (10% of demand against 25% at 96) + remove the spelling damage. +- Footprint starts at 81 GiB and peaks at 87 GiB over three server runs, 6 to 8 GiB above + the 11.8 + 0.336 per resident estimate, so the estimate needs that margin at this size. +- Speed was 11–18 tok/s, but the residual fit shared the GPU during these runs; a clean + speed run is queued (`jobs/after_b2.sh`). From ff7b7aff0dc5c49aeba303efe7939d886d2d51c3 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:30:49 -0800 Subject: [PATCH 69/74] v41_reap: local_eval.py --hot-tier DIR, hot experts as 2-bit codes plus a 1-bit residual; B2 result (64 hot at 3 bits level with 32 at FP4 at 92 GB) --- .../2026-11-08-v41-reap-recovery-log.md | 12 ++++++ .../2026-11-10-v41-wow-build-plan.md | 36 +++++++++++++++++ scripts/runpod/v41_reap/local_eval.py | 40 ++++++++++++------- scripts/runpod/v41_reap/routed_report.py | 5 ++- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 9121eb3..cc34499 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1071,3 +1071,15 @@ What it says: - Sampled scripts at the card's settings: 19 of 24 correct (144 residents 17, exact REAP-50 18). No failure is a misspelled name or syntax error; at 144 residents most were. - Speed not measured cleanly (the residual fit shared the GPU); queued. Rubric unchanged at 59. + +### 2026-11-12: third bit on the hottest experts (B2) + +- 1-bit residual fitted on the 2-bit base for the 192 survivors: weight error 0.084 to 0.030 + (gate, up), 0.047 to 0.024 (down). `local_eval.py --hot-tier DIR` runs hot experts as base + + residual. +- At 92 GB: 218 residents with the hottest 64 at 3 bits score 0.311 (0.215 without tool-use), + against 0.325 (0.219) for 217 with 32 at FP4 and 0.398 (0.254) for 250 at 2 bits. Paired + against FP4: −0.015 ± 0.008, −0.004 ± 0.003 without tool-use. The residual matches the FP4 + copy at equal memory and needs no second copy of the expert. +- 32 hot at 3 bits equals 32 at FP4; 96 hot adds nothing over 64. +- Rubric unchanged at 59: no item's own test was met. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index c409eb4..e27483c 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -799,3 +799,39 @@ sampling, 24 samples, drafter off, tau 0.3, 200 residents at 2 bits over the poo the 11.8 + 0.336 per resident estimate, so the estimate needs that margin at this size. - Speed was 11–18 tok/s, but the residual fit shared the GPU during these runs; a clean speed run is queued (`jobs/after_b2.sh`). + +### Third bit on the hottest experts (B2, 2026-11-12) + +A 1-bit E8 residual was fitted on the 2-bit base for the 192 survivors +(`keep_quant.py --residual-of quant/e8p-podimp --code-bits 8`, 310 s per layer, 33 GB). +Weighted weight error falls from 0.084 to 0.030 (gate and up) and from 0.047 to 0.024 (down). +A hot expert then runs base + residual, 13.8 MB instead of 18.8 MB for FP4 +(`local_eval.py --hot-tier DIR`). Same 92 GB budget and settings as B1b. + +| Variant | Residents | KL | KL, no tool-use | Top-1 | +|---|---:|---:|---:|---:| +| 250 at 2 bits (C1) | 250 | 0.398 | 0.254 | 84.4% | +| Hottest 32 at FP4 (B1b) | 217 | 0.325 | 0.219 | 85.6% | +| Hottest 32 at 3 bits | 234 | 0.327 | 0.223 | 85.3% | +| Hottest 64 at 3 bits | 218 | 0.311 | 0.215 | 85.9% | +| Hottest 96 at 3 bits | 202 | 0.308 | 0.222 | 85.8% | + +Paired differences, mean ± standard error (sessions better): + +| Pair | All 22 | Without tool-use (20) | +|---|---|---| +| 64 at 3 bits against 32 at FP4 | −0.015 ± 0.008 (16) | −0.004 ± 0.003 (14) | +| 32 at 3 bits against 32 at FP4 | +0.002 ± 0.004 (10) | +0.004 ± 0.004 (9) | +| 96 at 3 bits against 64 at 3 bits | −0.002 ± 0.011 (8) | +0.007 ± 0.004 (7) | +| 64 at 3 bits against 250 at 2 bits | −0.087 ± 0.034 (19) | −0.039 ± 0.009 (17) | +| 64 at 3 bits against all 384 at 2 bits (139 GB) | −0.079 ± 0.042 (14) | −0.020 ± 0.008 (12) | + +- The residual does the work of the FP4 copy at equal memory: level with it on the reading + without tool-use, slightly ahead with it. One more bit on a hot expert recovers what FP4 on + it recovers, because the memory saved goes to more hot experts or more residents. +- 64 hot experts per layer is the best of the three; 96 gives nothing more. +- Unlike an FP4 copy, the residual adds to the base the cache already holds. A hot expert + needs no second format in the slot array, and losing the residual leaves the expert running + at 2 bits. +- Run: 5,854 s, peak RSS 63 GB. Smoke (one session, 96 tokens, 144 residents): 1.521 at 2 + bits, 1.309 with 32 hot experts at 3 bits. diff --git a/scripts/runpod/v41_reap/local_eval.py b/scripts/runpod/v41_reap/local_eval.py index 3aa7b5e..de71c18 100644 --- a/scripts/runpod/v41_reap/local_eval.py +++ b/scripts/runpod/v41_reap/local_eval.py @@ -273,6 +273,9 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--hot", action="append", default=[], metavar="NAME=H", help="with --miss: the H hottest experts (--cache-warm order) that the checkpoint holds run from " "its FP4 weights while resident, the other residents from the codes") + p.add_argument("--hot-tier", default="fp4", metavar="fp4|DIR", + help="with --hot: what a hot expert runs. fp4, or its codes plus the residual in DIR, a keep_quant.py " + "--residual-of fit over the checkpoint's experts") p.add_argument("--cache-warm", type=Path, help="imatrix npz (reap__gate_norm_sum): warm-start each cache with the top-K") p.add_argument("--sidecar", type=Path, help="a sidecar_fit.py output directory, added to every variant's MoE output") p.add_argument("--sidecar-rank", type=int, help="use only the top-k sidecar components (stored largest first)") @@ -400,6 +403,12 @@ def simulate_cache(layer, ind, wts, k, tau, counts, resident=None): traces[v].extend((layer, *row) for row in found) return hi + def decode(z, proj: str, e: int): + """float32 weights of expert ``e`` of one keep_quant layer file.""" + vec = grids[int(z.get("code_bits", 16))][torch.as_tensor(z[f"{proj}_codes"][e].astype(np.int64), device=dev)] + s = torch.as_tensor(z[f"{proj}_scales"][e].astype(np.float32), device=dev) # vec: [out, in/8, 8] + return (vec * s.repeat_interleave(vec.shape[1] // s.shape[-1], dim=-1)[..., None]).flatten(-2) + def tier_weights(layer: int, e: int) -> tuple[dict, dict]: """(base, base + residual) bf16 weights of local expert ``e``, decoded once per layer.""" if cache["layer"] != layer: @@ -407,15 +416,7 @@ def tier_weights(layer: int, e: int) -> tuple[dict, dict]: cache["layer"] = layer cache["z"] = [layer_codes(s, layer, sorted(ckpt_survivors[str(layer)])) for s in tier_specs] if e not in cache: - tiers = [] - for z in cache["z"]: - grid = grids[int(z.get("code_bits", 16))] # the first E8P runs wrote no code_bits - tier = {} - for proj in ("w1", "w2", "w3"): - vec = grid[torch.as_tensor(z[f"{proj}_codes"][e].astype(np.int64), device=dev)] # [out, in/8, 8] - s = torch.as_tensor(z[f"{proj}_scales"][e].astype(np.float32), device=dev) - tier[proj] = (vec * s.repeat_interleave(vec.shape[1] // s.shape[-1], dim=-1)[..., None]).flatten(-2) - tiers.append(tier) + tiers = [{proj: decode(z, proj, e) for proj in ("w1", "w2", "w3")} for z in cache["z"]] lo = {p: w.bfloat16() for p, w in tiers[0].items()} hi = lo if len(tiers) == 1 else {p: (tiers[1][p] if copy_mode else lo_w + tiers[1][p]).bfloat16() for p, lo_w in tiers[0].items()} @@ -455,14 +456,22 @@ def dynamic_forward(self, x, image_mask=None): counts[i, 6] += float(w[drop].sum()) hi_mask = hi_mask & ~drop lo_w, hi_w = tier_weights(layer, i) - if hot_n.get(state["variant"]): # residents of the hot set run the checkpoint's FP4 expert + if hot_n.get(state["variant"]): # residents of the hot set run at --hot-tier precision if "hot" not in cache: cache["hot"] = {n: hot_set(warm[layer], n, own[str(layer)]) for n in set(hot_n.values())} if i in cache["hot"][hot_n[state["variant"]]]: - if ("fp4", i) not in cache: - mine = self.experts[own[str(layer)].index(i)] - cache["fp4", i] = {p: decode_fp4(getattr(mine, p).weight, getattr(mine, p).scale) for p in lo_w} - hi_w = cache["fp4", i] + if ("hot", i) not in cache: + j = own[str(layer)].index(i) # its place in the checkpoint, and in a fit over it + if args.hot_tier == "fp4": + mine = self.experts[j] + cache["hot", i] = {p: decode_fp4(getattr(mine, p).weight, getattr(mine, p).scale) for p in lo_w} + else: + if "residual" not in cache: + cache["residual"] = np.load(Path(args.hot_tier).expanduser() / f"layer-{layer:02d}.npz") + assert cache["residual"]["w1_codes"].shape[0] == len(own[str(layer)]) + cache["hot", i] = {p: (decode(cache["z"][0], p, i) + decode(cache["residual"], p, j)).bfloat16() + for p in lo_w} + hi_w = cache["hot", i] hot_slots[state["variant"]] += int(hi_mask.sum()) only = hi_proj.get(state["variant"]) if only is not None: # a finer rung: the residual of some projections only @@ -621,7 +630,8 @@ def before_moe(layer: int, block, key) -> None: "policy": policies.get(v, "lru") if v in cache_k else None, "copy_mode": copy_mode, "hi_units": hi_units, "n_bases": 0 if v in miss else kept, "pool": kept, "miss": miss.get(v), - "hot": hot_n.get(v, 0), "hot_slot_share": hot_slots[v] / max(tot[:2].sum() + tot[5], 1), + "hot": hot_n.get(v, 0), "hot_tier": "fp4" if args.hot_tier == "fp4" else "nested", + "hot_slot_share": hot_slots[v] / max(tot[:2].sum() + tot[5], 1), "tau_drop": tau_drop.get(v, 0.0), "hi_slot_share": float(tot[1] / (tot[:2].sum() + tot[5])), "drop_slot_share": float(tot[5] / (tot[:2].sum() + tot[5])), diff --git a/scripts/runpod/v41_reap/routed_report.py b/scripts/runpod/v41_reap/routed_report.py index 3802b7b..d2d04c6 100644 --- a/scripts/runpod/v41_reap/routed_report.py +++ b/scripts/runpod/v41_reap/routed_report.py @@ -60,8 +60,9 @@ def rows(eval_dir: Path, tok_s: float, bandwidth_gbs: float) -> list[dict]: "ssd_mb_per_tok": round(ssd / 1e6, 1), "ssd_gbs": round(ssd * tok_s / 1e9, 2), "read_gb_per_tok": round(read / 1e9, 2), "tok_s_bound": round(bandwidth_gbs * 1e9 / read, 1), } - if tiers.get("hot"): # a cache whose hottest experts are FP4 copies - mixed = (k - tiers["hot"]) * hi_units * UNIT_BYTES + tiers["hot"] * FP4_BYTES + if tiers.get("hot"): # a cache whose hottest experts are FP4 copies, or carry one more bit as a residual + hot_bytes = FP4_BYTES if tiers.get("hot_tier", "fp4") == "fp4" else (hi_units + 1) * UNIT_BYTES + mixed = (k - tiers["hot"]) * hi_units * UNIT_BYTES + tiers["hot"] * hot_bytes row |= {"hot": tiers["hot"], "hot_slots": round(tiers["hot_slot_share"], 3), "resident_gb": round(mixed * n_layers / 1e9, 1)} out.append(row) From 97062e1a46bdd0eef6aa15b4fe34c9ce9d9587ea Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:55:27 -0800 Subject: [PATCH 70/74] v41_build: resident experts from RAM by default (OMLX_LATTICE_SLOTS=auto); docs: 180 residents with the drafter (21 of 24 sampled scripts, median 32 tok/s, 88 GiB) --- .../2026-11-08-v41-reap-recovery-log.md | 8 ++++ .../2026-11-10-v41-wow-build-plan.md | 38 ++++++++++++++++ scripts/v41_build/omlx_lattice.py | 43 ++++++++++++++++--- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index cc34499..fd22be0 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1083,3 +1083,11 @@ What it says: copy at equal memory and needs no second copy of the expert. - 32 hot at 3 bits equals 32 at FP4; 96 hot adds nothing over 64. - Rubric unchanged at 59: no item's own test was met. + +### 2026-11-12: 180 residents with the drafter + +- Clean speed at 200 residents, drafter off: 21–28 tok/s, the same as at 144. +- 180 residents with the drafter on: 88 GiB peak, 22–41 tok/s (median 32), 21 of 24 sampled + scripts correct at the card's settings. Exact REAP-50 on stock oMLX: 18 of 24 at about 6 + tok/s. Candidate default for a 128 GB Mac. +- Rubric unchanged at 59: item 6 still needs the memory-pressure run. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index e27483c..dd204c6 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -835,3 +835,41 @@ Paired differences, mean ± standard error (sessions better): at 2 bits. - Run: 5,854 s, peak RSS 63 GB. Smoke (one session, 96 tokens, 144 residents): 1.521 at 2 bits, 1.309 with 32 hot experts at 3 bits. + +### Clean speed at 200 residents, and 180 residents with the drafter (2026-11-12) + +Same sampled script request, card sampling, tau 0.3, nothing else on the GPU. + +| Configuration | Footprint, peak | tok/s | Correct | +|---|---:|---:|---| +| 200 residents, drafter off | 87 GiB | 21–28 (3 samples) | 19 of 24 (above), 3 of 3 here | +| 180 residents, drafter on | 88 GiB | 22–41, median 32 (24 samples) | 21 of 24 (7, 8, 6) | +| 144 residents, drafter off (above) | 63 GiB | 22–27 | 17 of 24 | +| Exact REAP-50, stock offload (above) | | 5.6–6.3 | 18 of 24 | + +- More residents cost no speed: 200 residents decode as fast as 144. +- 180 residents with the drafter is the candidate default for a 128 GB Mac: every expert of + the full model reachable, 88 GiB, a median of 32 tok/s, and 21 of 24 sampled scripts + correct, against 18 of 24 at about 6 tok/s for the exact REAP-50 checkpoint on stock oMLX. + The three failures are a wrong listing, a wrong output format and one wrong keyword + argument; none is a misspelling. +- 21 against 18 of 24 is not a separable difference (Fisher p about 0.5); the claim is "no + worse, five times faster", not "better". + +### Slots from RAM (C3, 2026-11-12) + +`OMLX_LATTICE_SLOTS` now defaults to `auto` (`auto_slots` in `scripts/v41_build/omlx_lattice.py`): +a footprint budget of 70% of RAM (`OMLX_LATTICE_BUDGET_GIB` overrides it), less 20 GiB outside +the routed experts and 8 GiB when `model_settings.json` switches the drafter on, at 0.336 GiB +per resident. The constants are this M4 Max's measured peaks; other chips are unmeasured. + +| RAM | Residents, drafter off | Residents, drafter on | +|---:|---:|---:| +| 64 GB | 73 | 50 | +| 96 GB | 140 | 116 | +| 128 GB | 207 | 183 | + +- Below 120 residents the server prints that generated code was unreliable in our tests + (C2), so a 64 GB Mac starts with a warning and a 96 GB Mac should leave the drafter off. +- Checked on this Mac with the drafter on: 183 residents chosen, 83 GiB peak over two + requests, 39–41 tok/s, both scripts correct. diff --git a/scripts/v41_build/omlx_lattice.py b/scripts/v41_build/omlx_lattice.py index 9b9fd15..7c7dd15 100644 --- a/scripts/v41_build/omlx_lattice.py +++ b/scripts/v41_build/omlx_lattice.py @@ -27,8 +27,9 @@ directory, in file order; the first directory holds the loaded checkpoint's experts. ``router`` has every router row of the pool and ``mass`` (``reap__gate_norm_sum``) orders the starting residents. -Environment: ``OMLX_LATTICE_SLOTS`` (resident experts per layer, default 144) and the policy -settings of ``omlx_routed.py``. ``check`` compares one layer's decode step with dense matrices +Environment: ``OMLX_LATTICE_SLOTS`` (resident experts per layer; default ``auto``, from this +machine's RAM, see ``auto_slots``), ``OMLX_LATTICE_BUDGET_GIB`` (the footprint ``auto`` aims at, +default 70% of RAM) and the policy settings of ``omlx_routed.py``. ``check`` compares one layer's decode step with dense matrices decoded on the host, the prefill path with the decode path, and slots after installs with their stored bytes, then times the decode-step MoE. """ @@ -319,6 +320,27 @@ def wide_pool(path: Path, loaded: dict): return [Path(d).expanduser() for d, _ in spec["codes"]], order, full, where +FLOOR = 120 # residents per layer below which sampled code came out misspelled (build plan, C2) + + +def auto_slots(ram_bytes: int, drafter: bool, budget_gib: float | None = None) -> int: + """Resident experts per layer for a footprint of ``budget_gib`` (default: 70% of RAM). + + Measured on an M4 Max, 128 GB (2026-11-12): the server's footprint peaks near 20 GiB plus + 0.336 GiB per resident, and 8 GiB more with the drafter (87 GiB at 200 residents, 88 GiB at + 180 with the drafter). Other chips are unmeasured; the budget is the knob.""" + budget = 0.7 * ram_bytes / 2**30 if budget_gib is None else budget_gib + return int((budget - 20 - 8 * drafter) / 0.336) + + +def drafter_on(argv: list[str]) -> bool: + """Does ``--base-path``'s model_settings.json switch the drafter on for any model?""" + if "--base-path" not in argv: + return False + settings = Path(argv[argv.index("--base-path") + 1]).expanduser() / "model_settings.json" + return settings.exists() and any(m.get("mtp_enabled") for m in json.loads(settings.read_text())["models"].values()) + + def patch(spec: dict, root: Path, slots: int) -> None: full, state = {}, {} compact_patch(spec, full) @@ -329,9 +351,9 @@ def load_lattice(*args, **kwargs): config = model.language_model.layers[0].ffn.gate._config if root.is_file(): roots, order, wide, where = wide_pool(root, full) - state["cache"] = LatticeCache(model, config, roots, order, wide, slots, where) + state["cache"] = LatticeCache(model, config, roots, order, wide, min(slots, len(order[0])), where) else: - state["cache"] = LatticeCache(model, config, [root], spec["order"], full, slots) + state["cache"] = LatticeCache(model, config, [root], spec["order"], full, min(slots, len(spec["order"][0]))) full.clear() return model, processor @@ -451,8 +473,17 @@ def step(n): a = p.parse_args(sys.argv[2:]) check(a.config.expanduser(), a.codes.expanduser()) else: - patch(json.loads(Path(os.environ["OMLX_COMPACT"]).read_text()), Path(os.environ["OMLX_LATTICE"]).expanduser(), - int(os.environ.get("OMLX_LATTICE_SLOTS", "144"))) + assert auto_slots(128 << 30, True) == 183 and auto_slots(128 << 30, False, 87) == 199 + slots = os.environ.get("OMLX_LATTICE_SLOTS", "auto") + if slots == "auto": + budget = os.environ.get("OMLX_LATTICE_BUDGET_GIB") + slots = auto_slots(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE"), drafter_on(sys.argv), + float(budget) if budget else None) + slots = int(slots) + print(f"lattice cache: {slots} resident experts per layer" + ( + f"; below {FLOOR}, generated code was unreliable in our tests" if slots < FLOOR else ""), file=sys.stderr) + assert slots >= 6, "not enough memory for the router's top-6" + patch(json.loads(Path(os.environ["OMLX_COMPACT"]).read_text()), Path(os.environ["OMLX_LATTICE"]).expanduser(), slots) from omlx.cli import main sys.exit(main()) From f2a55a673074bf9751ccbaf69eac8c6d0bd55b8b Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:56:01 -0800 Subject: [PATCH 71/74] docs: slots-from-RAM table, 64 GB with the drafter is 49 --- docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index dd204c6..e781371 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -865,7 +865,7 @@ per resident. The constants are this M4 Max's measured peaks; other chips are un | RAM | Residents, drafter off | Residents, drafter on | |---:|---:|---:| -| 64 GB | 73 | 50 | +| 64 GB | 73 | 49 | | 96 GB | 140 | 116 | | 128 GB | 207 | 183 | From 15a08a8594a0ce233520be21318ffcea30198d89 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:38:02 -0800 Subject: [PATCH 72/74] v41_build: slim_checkpoint.py and run_v41.sh, one command from a pool bundle (366 GB to 226 GB checkpoint, relative pool paths); verified from a clean oMLX directory --- .../2026-11-08-v41-reap-recovery-log.md | 10 ++ .../2026-11-10-v41-wow-build-plan.md | 21 ++++ scripts/v41_build/omlx_lattice.py | 13 ++- scripts/v41_build/run_v41.sh | 35 ++++++ scripts/v41_build/slim_checkpoint.py | 102 ++++++++++++++++++ 5 files changed, 177 insertions(+), 4 deletions(-) create mode 100755 scripts/v41_build/run_v41.sh create mode 100644 scripts/v41_build/slim_checkpoint.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index fd22be0..1a39bcf 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -1091,3 +1091,13 @@ What it says: scripts correct at the card's settings. Exact REAP-50 on stock oMLX: 18 of 24 at about 6 tok/s. Candidate default for a 128 GB Mac. - Rubric unchanged at 59: item 6 still needs the memory-pressure run. + +### 2026-11-12: slots from RAM, slim checkpoint, one command (C3, C4) + +- `OMLX_LATTICE_SLOTS=auto` is the default: 70% of RAM less 20 GiB and 8 GiB for the drafter, + at 0.336 GiB per resident (183 with the drafter on this Mac); a warning below 120 residents. +- `slim_checkpoint.py`: 365.8 GB to 225.9 GB, bytes checked. Bundle of 340 GiB with relative + paths; `run_v41.sh BUNDLE` serves it, installing oMLX at the pinned commit on the first run. + Verified from an empty oMLX directory: 183 residents, 83 GiB, 38–40 tok/s. +- `pool384/fp4-removed` deleted (Jack confirmed), 134 GB freed. +- Rubric unchanged at 59: none of this is a rubric item's test. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index e781371..806af6a 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -873,3 +873,24 @@ per resident. The constants are this M4 Max's measured peaks; other chips are un (C2), so a 64 GB Mac starts with a warning and a 96 GB Mac should leave the drafter off. - Checked on this Mac with the drafter on: 183 residents chosen, 83 GiB peak over two requests, 39–41 tok/s, both scripts correct. + +### One directory, one command (C4, 2026-11-12) + +- `scripts/v41_build/slim_checkpoint.py SRC DST` writes a compact model directory as real + files without the routed experts its loader never reads: 365.8 GB to 225.9 GB for the + lattice model (the two 101.5 GB Engram shards and three others lose nothing and are cloned, + 43 shards are rewritten and read back against the source by CRC). The six stub experts per + layer, the drafter and its experts stay. +- Bundle layout, 340 GiB (`~/Models/v41-reap-run/bundle`): `models/v41-lattice` (slim + checkpoint), `codes/kept` and `codes/removed` (the 2-bit codes of all 384 experts, unchanged), + `ids-kept.json`, `ids-removed.json`, `router.safetensors`, `mass.npz`, and `pool.json`, whose + paths are now relative to it. +- `scripts/v41_build/run_v41.sh BUNDLE [PORT]` checks the machine, clones oMLX at the pinned + commit and builds its venv with uv on the first run, writes `base/model_settings.json` + (drafter on from 128 GB of RAM) and starts the server; the resident count comes from RAM. +- Checked on this Mac: with the existing oMLX install, and again from an empty oMLX directory + (fresh clone at `4213c328`, `uv sync`): 183 residents, 83 GiB, 38–40 tok/s, 2 of 3 sampled + scripts correct in each run. +- `pool384/fp4-removed` (134 GB) was deleted with Jack's confirmation; the hot tier uses + survivors only. Not done: publishing (needs Jack's OK; the Mac's uplink is 3 MB/s, so an + upload would run from a pod) and a model card for the bundle. diff --git a/scripts/v41_build/omlx_lattice.py b/scripts/v41_build/omlx_lattice.py index 7c7dd15..e9100a6 100644 --- a/scripts/v41_build/omlx_lattice.py +++ b/scripts/v41_build/omlx_lattice.py @@ -26,6 +26,7 @@ ``IDS.json`` (``{"survivors": {layer: [...]}}``) gives the pool number of each expert of its directory, in file order; the first directory holds the loaded checkpoint's experts. ``router`` has every router row of the pool and ``mass`` (``reap__gate_norm_sum``) orders the starting residents. +Paths in the file may be relative to it. Environment: ``OMLX_LATTICE_SLOTS`` (resident experts per layer; default ``auto``, from this machine's RAM, see ``auto_slots``), ``OMLX_LATTICE_BUDGET_GIB`` (the footprint ``auto`` aims at, @@ -307,17 +308,21 @@ def wide_pool(path: Path, loaded: dict): """(directories, order, router rows, where) of the pool file ``path``. ``loaded`` is the router of the loaded checkpoint, whose rows must be the first directory's.""" spec = json.loads(path.read_text()) - ids = [json.loads(Path(f).expanduser().read_text())["survivors"] for _, f in spec["codes"]] + + def at(p: str) -> Path: # paths in the pool file are relative to it + return path.parent / Path(p).expanduser() + + ids = [json.loads(at(f).read_text())["survivors"] for _, f in spec["codes"]] where = [{e: (d, j) for d, per in enumerate(ids) for j, e in enumerate(per[str(i)])} for i in range(len(ids[0]))] - mass = np.load(Path(spec["mass"]).expanduser())["reap__gate_norm_sum"] + mass = np.load(at(spec["mass"]))["reap__gate_norm_sum"] assert all(sorted(w) == list(range(mass.shape[1])) for w in where), "the directories do not cover the pool once" - router = mx.load(str(Path(spec["router"]).expanduser())) + router = mx.load(str(at(spec["router"]))) full = {(i, f): router[f"layers.{i}.ffn.gate.{f}"] for i in range(len(where)) for f in ("weight", "bias")} for (i, f), rows in loaded.items(): if f in ("weight", "bias"): assert mx.array_equal(full[i, f][mx.array(ids[0][str(i)])], rows), f"layer {i}: router {f} differs" order = [np.argsort(-row, kind="stable").tolist() for row in mass] - return [Path(d).expanduser() for d, _ in spec["codes"]], order, full, where + return [at(d) for d, _ in spec["codes"]], order, full, where FLOOR = 120 # residents per layer below which sampled code came out misspelled (build plan, C2) diff --git a/scripts/v41_build/run_v41.sh b/scripts/v41_build/run_v41.sh new file mode 100755 index 0000000..276404e --- /dev/null +++ b/scripts/v41_build/run_v41.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Serve DeepSeek-V4.1-Flash-1017 from a pool bundle with the no-wait expert cache, under oMLX at a +# pinned commit: every one of the 384 experts per layer reachable, as many resident as RAM allows. +# +# run_v41.sh BUNDLE [PORT] +# +# BUNDLE holds models/v41-lattice (slim_checkpoint.py output), pool.json and what it names. The +# first run clones oMLX into OMLX_DIR (default ~/.cache/v41/omlx) and builds its venv with uv. +# OMLX_LATTICE_SLOTS and OMLX_LATTICE_BUDGET_GIB override the resident count (omlx_lattice.py). +set -euo pipefail +REPO=$(cd "$(dirname "$0")/../.." && pwd) +B=$(cd "${1:?usage: run_v41.sh BUNDLE [PORT]}" && pwd); PORT=${2:-8017} +OMLX=${OMLX_DIR:-$HOME/.cache/v41/omlx}; PIN=4213c3284abc7b3746a91464a7af7ccc3732b759 +[ "$(uname -sm)" = "Darwin arm64" ] || { echo "needs an Apple-silicon Mac" >&2; exit 1; } +RAM=$(( $(sysctl -n hw.memsize) >> 30 )) +[ "$RAM" -ge 96 ] || echo "warning: $RAM GB of RAM; generated code was unreliable below 96 GB in our tests" >&2 +for f in models/v41-lattice/compact-map.json pool.json; do + [ -f "$B/$f" ] || { echo "$B/$f is missing" >&2; exit 1; } +done +if [ ! -f "$B/base/model_settings.json" ]; then # the drafter costs 8 GiB: on from 128 GB of RAM + mkdir -p "$B/base" + [ "$RAM" -ge 120 ] && MTP=', "mtp_enabled": true, "mtp_fixed_depth": 3' || MTP= + echo "{\"version\": 2, \"models\": {\"v41-lattice\": {\"deepseek_v41_engram_ssd_offload\": true$MTP}}}" > "$B/base/model_settings.json" +fi +if [ ! -x "$OMLX/.venv/bin/python" ]; then + command -v git > /dev/null && command -v uv > /dev/null || { echo "needs git and uv (https://docs.astral.sh/uv/)" >&2; exit 1; } + [ -d "$OMLX/.git" ] || git clone --quiet https://github.com/jundot/omlx "$OMLX" + git -C "$OMLX" checkout --quiet "$PIN" + (cd "$OMLX" && uv sync --quiet) +fi +echo "endpoint http://127.0.0.1:$PORT/v1, model v41-lattice" >&2 +cd "$OMLX" +OMLX_COMPACT=$B/models/v41-lattice/compact-map.json OMLX_LATTICE=$B/pool.json \ + exec .venv/bin/python "$REPO/scripts/v41_build/omlx_lattice.py" serve --model-dir "$B/models" --base-path "$B/base" \ + --port "$PORT" --memory-guard-gb $((RAM * 3 / 4)) --max-concurrent-requests 1 --no-cache --headless diff --git a/scripts/v41_build/slim_checkpoint.py b/scripts/v41_build/slim_checkpoint.py new file mode 100644 index 0000000..1a97267 --- /dev/null +++ b/scripts/v41_build/slim_checkpoint.py @@ -0,0 +1,102 @@ +"""A compact model directory as real files, without the routed experts its loader never reads. + +SRC is an ``omlx_compact.py make`` directory: config, tensor index and ``compact-map.json`` beside +links to a checkpoint's shards. The lattice cache (``omlx_lattice.py``) loads only the few experts +the map names and takes every other routed expert from its codes, so the rest of the FP4 experts +are dead weight in a download. DST gets every file of SRC; a shard that loses nothing is cloned +(``cp -c``, free on APFS) and the others are rewritten with the tensors that stay, names and bytes +untouched. Each rewritten shard is read back and compared with the source by CRC. Standard library +only. + + python slim_checkpoint.py SRC DST +""" + +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import zlib +from pathlib import Path + +EXPERT = re.compile(r"layers\.\d+\.ffn\.experts\.\d+\.") # a routed expert of a decoder layer (not the drafter's) +CHUNK = 1 << 24 + + +def header(fd: int) -> tuple[dict, int]: + (size,) = struct.unpack(" int: + """CRC of ``size`` bytes at ``begin``; with ``out``, the bytes are also written to it.""" + value, done = 0, 0 + while done < size: + chunk = os.pread(fd, min(size - done, CHUNK), begin + done) + assert chunk, "truncated shard" + value, done = zlib.crc32(chunk, value), done + len(chunk) + if out: + out.write(chunk) + return value + + +def slim(src: Path, dst: Path, keep: set[str]) -> tuple[int, int]: + """Write shard ``src`` to ``dst`` without its routed experts outside ``keep``. (bytes in, bytes out).""" + fd = os.open(src, os.O_RDONLY) + entries, start = header(fd) + names = [n for n in entries if n != "__metadata__" and (n in keep or not EXPERT.match(n))] + if len(names) == len(entries) - ("__metadata__" in entries): + os.close(fd) + if subprocess.run(["cp", "-c", str(src), str(dst)], capture_output=True, check=False).returncode: # not APFS: a plain copy + shutil.copyfile(src, dst) + return src.stat().st_size, src.stat().st_size + names.sort(key=lambda n: entries[n]["data_offsets"][0]) + out_header, offset = {k: entries[k] for k in ("__metadata__",) if k in entries}, 0 + for n in names: + begin, end = entries[n]["data_offsets"] + out_header[n] = {**entries[n], "data_offsets": [offset, offset + end - begin]} + offset += end - begin + blob = json.dumps(out_header, separators=(",", ":")).encode() + blob += b" " * (-len(blob) % 8) + with open(dst, "wb") as out: + out.write(struct.pack(" None: + src, dst = Path(sys.argv[1]).expanduser(), Path(sys.argv[2]).expanduser() + spec = json.loads((src / "compact-map.json").read_text()) + keep = set(spec["alias"].values()) # stored names of the experts the loader reads + dst.mkdir(parents=True) + total_in = total_out = 0 + for item in sorted(src.iterdir()): + real = item.resolve() + if item.name in (".cache", "SHA256SUMS"): # download bookkeeping and sums of the full shards + continue + if item.name == "compact-map.json": + (dst / item.name).write_text(json.dumps(spec | {"source": "."})) + elif item.name.endswith(".safetensors") and item.name.startswith("model-"): + a, b = slim(real, dst / item.name, keep) + total_in, total_out = total_in + a, total_out + b + print(f"{item.name}: {a / 1e9:.2f} GB -> {b / 1e9:.2f} GB", flush=True) + elif real.is_dir(): + shutil.copytree(real, dst / item.name) + else: + shutil.copyfile(real, dst / item.name) + print(f"{dst}: {total_in / 1e9:.1f} GB -> {total_out / 1e9:.1f} GB") + + +if __name__ == "__main__": + main() From 5f288930de67dc421baca64e149c5cf5ce89de7e Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:40:49 -0800 Subject: [PATCH 73/74] ramp: e8_1bit_m1, few-token kernels for the 1-bit E8 residual of a nested tier, tested against a float64 decode --- .../2026-11-10-v41-wow-build-plan.md | 16 +++ src/ramp/kernels/e8_1bit_m1.metal | 90 +++++++++++++++++ src/ramp/kernels/e8_1bit_m1.py | 97 +++++++++++++++++++ tests/test_e8_1bit_m1.py | 66 +++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 src/ramp/kernels/e8_1bit_m1.metal create mode 100644 src/ramp/kernels/e8_1bit_m1.py create mode 100644 tests/test_e8_1bit_m1.py diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 806af6a..603d429 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -894,3 +894,19 @@ per resident. The constants are this M4 Max's measured peaks; other chips are un - `pool384/fp4-removed` (134 GB) was deleted with Jack's confirmation; the hot tier uses survivors only. Not done: publishing (needs Jack's OK; the Mac's uplink is 3 MB/s, so an upload would run from a pod) and a model card for the bundle. + +### Residual kernel for the hot tier (B3, first part, 2026-11-12) + +- `src/ramp/kernels/e8_1bit_m1.metal` and `e8_1bit_m1.py`: the token and per-route layouts of + `e8p_m1` over 8-bit E8 codes, for a residual added to the 2-bit output. A route whose index + is `NO_RESIDUAL` writes zeros, so one call serves a token whose routes are partly hot. +- `tests/test_e8_1bit_m1.py` (19 cases): both layouts against a float64 decode of the same + codes, fp32 within 1e-5 and bf16 within 2e-2 of the largest output, with routes that have + no residual, a ragged last tile and the model's group size; also passes under + `MTL_SHADER_VALIDATION=1`. +- Timed alone at the model's sizes (one token, six routes, 200 interleaved runs, M4 Max): gate + 0.217 ms against 0.227 ms for the 2-bit kernel, down 0.248 against 0.252; with two of six + routes hot 0.185 and 0.194. Single calls are launch-bound, so the cost inside the decode step + is still to be measured. +- Not done: the hot residual arrays, the per-slot hot index and the memory accounting in + `omlx_lattice.py`. diff --git a/src/ramp/kernels/e8_1bit_m1.metal b/src/ramp/kernels/e8_1bit_m1.metal new file mode 100644 index 0000000..2fc8efc --- /dev/null +++ b/src/ramp/kernels/e8_1bit_m1.metal @@ -0,0 +1,90 @@ +// Few-token gather matvec for 8-bit E8 codes (the 1-bit residual of a nested +// tier), the companion of e8p_m1.metal: one threadgroup column per route, +// route r multiplies activation row r / X_DIV by expert rhs_indices[r]. A +// route whose index is 0xFFFFFFFF has no residual and writes zeros. +// +// A codeword is one byte that indexes a 256-entry table of 8 values. The +// table arrives with nibbles k and k+4 holding dims 2k and 2k+1 (see +// pair_packed in e8_1bit_m1.py), so each threadgroup decodes it into half2 +// pairs once and a codeword then costs one threadgroup read. A lane takes +// CHUNK (4 or 8) consecutive codewords with one 4- or 8-byte load per row. +uint simd_lane = thread_index_in_simdgroup; +uint simd_id = simdgroup_index_in_threadgroup; +uint tg_lane = thread_position_in_threadgroup.x; +uint row_tile = threadgroup_position_in_grid.y; +uint route = threadgroup_position_in_grid.z; + +uint expert = uint(rhs_indices[route]); +uint out_dim = uint(codes_shape[1]); +uint codewords = uint(codes_shape[2]); +uint groups = uint(scales_shape[2]); +uint codewords_per_group = codewords / groups; +uint x_row = (route / uint(X_DIV)) * codewords * 8u; +uint row0 = (row_tile * uint(SIMDGROUPS) + simd_id) * uint(ROWS_PER_SG); + +if (expert == 0xFFFFFFFFu) { + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + if (simd_lane == 0u && row0 + r < out_dim) { + out[route * out_dim + row0 + r] = OUT_T(0.0f); + } + } + return; +} + +// (1024 + n) * 0.5 - 516 = (n - 8) / 2, exact in half. +threadgroup uint4 decoded[256]; +for (uint idx = tg_lane; idx < 256u; idx += threads_per_threadgroup.x) { + uint packed = codebook[idx]; + uint4 entry; + for (uint k = 0; k < 4u; ++k) { + half2 h = as_type(((packed >> (4u * k)) & 0x000F000Fu) | 0x64006400u); + entry[k] = as_type(fma(h, half2(0.5h), half2(-516.0h))); + } + decoded[idx] = entry; +} +threadgroup_barrier(mem_flags::mem_threadgroup); + +float acc[ROWS_PER_SG]; +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + acc[r] = 0.0f; +} + +for (uint cw0 = simd_lane * uint(CHUNK); cw0 < codewords; cw0 += 32u * uint(CHUNK)) { + float2 xv[CHUNK * 4]; + for (uint i = 0; i < uint(CHUNK); ++i) { + for (uint k = 0; k < 4u; ++k) { + uint xi = x_row + (cw0 + i) * 8u + 2u * k; + xv[i * 4u + k] = float2(float(x[xi]), float(x[xi + 1u])); + } + } + uint group = cw0 / codewords_per_group; + for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + if (row >= out_dim) { + break; + } + uint base = expert * out_dim + row; + uint2 words = uint2(0u); + if (uint(CHUNK) == 8u) { + words = ((const device uint2*)(codes + base * codewords))[cw0 / 8u]; + } else { + words.x = ((const device uint*)(codes + base * codewords))[cw0 / 4u]; + } + float part = 0.0f; + for (uint i = 0; i < uint(CHUNK); ++i) { + uint4 entry = decoded[(words[i >> 2u] >> ((i & 3u) * 8u)) & 0xFFu]; + for (uint k = 0; k < 4u; ++k) { + part += dot(float2(as_type(entry[k])), xv[i * 4u + k]); + } + } + acc[r] += part * float(scales[base * groups + group]); + } +} + +for (uint r = 0; r < uint(ROWS_PER_SG); ++r) { + uint row = row0 + r; + float total = simd_sum(acc[r]); + if (simd_lane == 0u && row < out_dim) { + out[route * out_dim + row] = OUT_T(total); + } +} diff --git a/src/ramp/kernels/e8_1bit_m1.py b/src/ramp/kernels/e8_1bit_m1.py new file mode 100644 index 0000000..ad882d4 --- /dev/null +++ b/src/ramp/kernels/e8_1bit_m1.py @@ -0,0 +1,97 @@ +"""Few-token kernels for 8-bit E8 codes, the 1-bit residual of a nested tier. + +The companion of ``e8p_m1``: the same two layouts over ``uint8`` codes, for a residual that is +added to the output of the 16-bit base. A route whose index is ``NO_RESIDUAL`` writes zeros, so +one call serves a token whose routes are partly at the higher tier. + +- :func:`e8_1bit_m1_token_layout` for gate/up (one activation row, top-k experts). +- :func:`e8_1bit_m1_per_route` for down (one activation row per route). +""" + +from __future__ import annotations + +from functools import cache +from pathlib import Path + +import numpy as np + +from ramp.kernels.e8p_m1 import _chunk + +__all__ = ["NO_RESIDUAL", "e8_1bit_m1_per_route", "e8_1bit_m1_token_layout", "pair_packed"] + +NO_RESIDUAL = 0xFFFFFFFF + + +def pair_packed(packed: np.ndarray) -> np.ndarray: + """The packed 1-bit table (nibble d is dim d) with dims 2k and 2k+1 in nibbles k and k+4, + the layout the kernel decodes into half2 pairs.""" + + p, out = np.asarray(packed, dtype=np.uint32), np.zeros(256, np.uint32) + for k in range(4): + out |= ((p >> np.uint32(8 * k)) & np.uint32(0xF)) << np.uint32(4 * k) + out |= ((p >> np.uint32(8 * k + 4)) & np.uint32(0xF)) << np.uint32(16 + 4 * k) + return out + + +@cache +def _kernel(): + import mlx.core as mx + + return mx.fast.metal_kernel( + name="ramp_e8_1bit_m1", + input_names=["x", "codes", "scales", "codebook", "rhs_indices"], + output_names=["out"], + source=(Path(__file__).resolve().parent / "e8_1bit_m1.metal").read_text(), + ) + + +def _launch(x, codes, scales, codebook, rhs_indices, *, output_dims, rows_per_simdgroup, simdgroups, x_div): + import mlx.core as mx + + from ramp.kernels.vq_qmv import _qmv_output_dtype + + if codes.dtype != mx.uint8: + raise ValueError(f"8-bit E8 codes must be uint8, got {codes.dtype}") + if rows_per_simdgroup < 1 or simdgroups < 1 or simdgroups > 32: + raise ValueError("rows_per_simdgroup must be >= 1 and simdgroups in 1..32") + routes, rows_per_tile = int(rhs_indices.size), rows_per_simdgroup * simdgroups + output_dtype = _qmv_output_dtype(x.dtype) + (out,) = _kernel()( + inputs=[mx.contiguous(x), codes, scales, codebook, rhs_indices.reshape(-1)], + template=[ + ("ROWS_PER_SG", rows_per_simdgroup), + ("SIMDGROUPS", simdgroups), + ("CHUNK", _chunk(int(codes.shape[2]), int(scales.shape[2]))), + ("X_DIV", int(x_div)), + ("OUT_T", output_dtype), + ], + grid=(32 * simdgroups, (output_dims + rows_per_tile - 1) // rows_per_tile, routes), + threadgroup=(32 * simdgroups, 1, 1), + output_shapes=[(1, routes, output_dims)], + output_dtypes=[output_dtype], + ) + return out + + +def e8_1bit_m1_token_layout(x, codes, scales, codebook, rhs_indices, *, output_dims: int, + rows_per_simdgroup: int = 4, simdgroups: int = 1): + """Token ``t``'s row against its ``top_k`` experts: ``[tokens, top_k, out]``.""" + + if x.ndim != 2 or rhs_indices.ndim != 2 or rhs_indices.shape[0] != x.shape[0]: + raise ValueError("token-layout E8 needs x [tokens, in] and rhs_indices [tokens, top_k]") + tokens, top_k = int(rhs_indices.shape[0]), int(rhs_indices.shape[1]) + out = _launch(x, codes, scales, codebook, rhs_indices, output_dims=output_dims, + rows_per_simdgroup=rows_per_simdgroup, simdgroups=simdgroups, x_div=top_k) + return out.reshape(tokens, top_k, output_dims) + + +def e8_1bit_m1_per_route(x, codes, scales, codebook, rhs_indices, *, output_dims: int, + rows_per_simdgroup: int = 4, simdgroups: int = 1): + """Route ``r``'s activation row against expert ``rhs_indices[r]``: ``[routes, out]``.""" + + routes = int(rhs_indices.size) + if x.ndim != 2 or x.shape[0] != routes: + raise ValueError(f"per-route E8 needs x [{routes}, in], got {tuple(x.shape)}") + out = _launch(x, codes, scales, codebook, rhs_indices, output_dims=output_dims, + rows_per_simdgroup=rows_per_simdgroup, simdgroups=simdgroups, x_div=1) + return out.reshape(routes, output_dims) diff --git a/tests/test_e8_1bit_m1.py b/tests/test_e8_1bit_m1.py new file mode 100644 index 0000000..5b7380b --- /dev/null +++ b/tests/test_e8_1bit_m1.py @@ -0,0 +1,66 @@ +"""The 8-bit E8 residual kernels against a float64 decode of the same codes.""" + +from __future__ import annotations + +import numpy as np +import pytest + +mx = pytest.importorskip("mlx.core") + +from keep.vq.e8 import e8_1bit_grid, e8_1bit_packed +from ramp.kernels import e8_1bit_m1 + +pytestmark = pytest.mark.skipif(not (getattr(mx, "metal", None) and mx.metal.is_available()), reason="needs a Metal GPU") +NONE = e8_1bit_m1.NO_RESIDUAL + + +def _layer(experts, out, inp, group_size, seed): + rng = np.random.default_rng(seed) + codes = rng.integers(0, 256, (experts, out, inp // 8), dtype=np.uint8) + scales = rng.uniform(0.01, 0.2, (experts, out, inp // group_size)).astype(np.float16) + dense = e8_1bit_grid().astype(np.float64)[codes].reshape(experts, out, inp) + dense *= np.repeat(scales.astype(np.float64), group_size, axis=-1) + return mx.array(codes), mx.array(scales), dense + + +def _close(got, want, tol): + got = np.array(got.astype(mx.float32)).astype(np.float64) + assert got.shape == want.shape, (got.shape, want.shape) + err = np.abs(got - want).max() / np.abs(want).max() + assert err <= tol, err + + +def test_pair_packed_keeps_every_value(): + pairs = e8_1bit_m1.pair_packed(e8_1bit_packed()) + nib = lambda p, n: ((p >> (4 * n)) & 0xF).astype(np.float64) + back = np.stack([(nib(pairs, k if d % 2 == 0 else k + 4) - 8) / 2 for d in range(8) for k in [d // 2]], axis=1) + np.testing.assert_array_equal(back, e8_1bit_grid()) + + +# group 32 -> 4 codewords per group (4-byte loads); 64 -> 8. inp=768 makes lanes take more than +# one step; out=93 leaves a ragged last tile. +@pytest.mark.parametrize("group_size,inp,out", [(32, 128, 96), (64, 768, 93), (256, 4096, 64)]) +@pytest.mark.parametrize("rows_per_simdgroup,simdgroups", [(1, 1), (8, 1), (4, 4)]) +@pytest.mark.parametrize("dtype,tol", [(mx.float32, 1e-5), (mx.bfloat16, 2e-2)]) +def test_both_layouts_match_the_dense_decode(group_size, inp, out, rows_per_simdgroup, simdgroups, dtype, tol): + codes, scales, dense = _layer(6, out, inp, group_size, seed=3) + book = mx.array(e8_1bit_m1.pair_packed(e8_1bit_packed())) + kw = {"output_dims": out, "rows_per_simdgroup": rows_per_simdgroup, "simdgroups": simdgroups} + rng = np.random.default_rng(4) + rhs = np.array([[5, 0, 3], [NONE, 2, 2], [1, NONE, 4]], dtype=np.uint32) + + def rows(n): # values as the kernel sees them, after rounding to dtype + x = mx.array(rng.normal(size=(n, inp)).astype(np.float32)).astype(dtype) + return x, np.array(x.astype(mx.float32)).astype(np.float64) + + def ref(x_row, e): + return np.zeros(out) if e == NONE else x_row @ dense[e].T + + x, xn = rows(3) + got = e8_1bit_m1.e8_1bit_m1_token_layout(x, codes, scales, book, mx.array(rhs), **kw) + assert got.dtype == dtype + _close(got, np.stack([[ref(xn[t], e) for e in rhs[t]] for t in range(3)]), tol) + + x, xn = rows(9) + got = e8_1bit_m1.e8_1bit_m1_per_route(x, codes, scales, book, mx.array(rhs.reshape(-1)), **kw) + _close(got, np.stack([ref(xn[r], e) for r, e in enumerate(rhs.reshape(-1))]), tol) From 61abef8a16b2495fcfa4611566dac9c5282d2559 Mon Sep 17 00:00:00 2001 From: Jack Mazac <114446521+jackmaxwil@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:22:44 -0800 Subject: [PATCH 74/74] v41_build: d3_pressure.py, decode speed under memory pressure (33.5 tok/s with 40 GB held, 36.7 before); run_v41.sh sets tau 0.3; rubric item 6 to 10 --- .../2026-11-08-v41-reap-recovery-log.md | 17 +++- .../2026-11-10-v41-wow-build-plan.md | 29 ++++++ scripts/v41_build/d3_pressure.py | 95 +++++++++++++++++++ scripts/v41_build/run_v41.sh | 3 +- 4 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 scripts/v41_build/d3_pressure.py diff --git a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md index 1a39bcf..4d9b239 100644 --- a/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md +++ b/docs/deepseek-v4-flash/2026-11-08-v41-reap-recovery-log.md @@ -600,11 +600,11 @@ to 100; partial credit only for measured progress). | 3 | Offload cache simulation and cost model | 10 | 10 | `residual_cache` (unit-tested), `routed_report.py` | | 4 | Core result: routed beats the static frontier at equal resident memory, SSD rate sustainable | 25 | 20 | E1 (2026-11-10): an expert cache with re-route beats the static set by 0.056 at 144 and 0.231 at 96, intervals clear of zero, 0.45 and 1.72 GB/s at 30 tok/s; inside a pool of 192, 22 sessions, simulated reads. Pool of 384 (2026-11-10): a cache of 144 beats the static 144 by 0.309 (21 of 22 sessions) and the exact REAP-50 checkpoint by 0.12, at 0.7 GB/s for tau 0.3. The rest needs the pod confirmation (E7) | | 5 | Unification: per-token tier 0 + expert pruning in one policy, frontier figure | 10 | 0 | tier 0 measured free at 7.7% of slots; no joint policy or figure yet | -| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 8 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max over the full pool of 384 with 144 2-bit lattice residents at 23–27 tok/s (17–19 with thinking on), 80 MB of real reads per token, 63 GiB. The rest needs memory pressure | +| 6 | Real Apple Silicon streaming measured, not simulated | 10 | 10 | 2026-11-10: the no-wait cache runs under oMLX on the M4 Max over the full pool of 384 with 144 2-bit lattice residents at 23–27 tok/s (17–19 with thinking on), 80 MB of real reads per token, 63 GiB. Memory pressure (2026-11-12): with another process holding 40 GB of incompressible memory and 20 GB pushed to swap, the server keeps its 83 GiB, answers all 61 requests and decodes at 33.5 tok/s against 36.7 before | | 7 | Learned tier policy or trained nested codes | 10 | 0 | pod work, after 4 | | 8 | Generality: a second MoE plus downstream benchmarks | 10 | 0 | | | 9 | Paper draft | 5 | 1 | framing and experiment plan in this log | -| | **Total** | 100 | **59** | | +| | **Total** | 100 | **61** | | ### 2026-11-09: prior art, plan, restart @@ -1101,3 +1101,16 @@ What it says: Verified from an empty oMLX directory: 183 residents, 83 GiB, 38–40 tok/s. - `pool384/fp4-removed` deleted (Jack confirmed), 134 GB freed. - Rubric unchanged at 59: none of this is a rubric item's test. + +### 2026-11-12: memory pressure (D3) + +- `scripts/v41_build/d3_pressure.py`: requests back to back while a second process takes + incompressible memory in 4 GB steps to 40 GB, holds it and lets it go. Server: the bundle + through `run_v41.sh`, 183 residents, drafter on, 83 GiB. +- At tau 0.3 (the shipped setting): 36.7 tok/s before, 29.5 and 32.8 during the steps, 33.5 + while 40 GB are held, 35.9 after. 61 requests, no error. The server's footprint stays at 83 + GiB, swap goes from 5 to 25 GB, the pressure level reaches "warning", never "critical". +- The first run used tau 0.2 by mistake: `run_v41.sh` did not set the threshold and the code's + default is 0.2, six times the SSD reads (33.6 MB per token against 1.5 to 5.5). Fixed in the + launcher. That run: 39.8 before, 32.2 held, 31.8 after. +- Rubric item 6 moves from 8 to 10. Total 61. diff --git a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md index 603d429..e7ad151 100644 --- a/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md +++ b/docs/deepseek-v4-flash/2026-11-10-v41-wow-build-plan.md @@ -910,3 +910,32 @@ per resident. The constants are this M4 Max's measured peaks; other chips are un is still to be measured. - Not done: the hot residual arrays, the per-slot hot index and the memory accounting in `omlx_lattice.py`. + +### Memory pressure (D3, 2026-11-12) + +`scripts/v41_build/d3_pressure.py` sends the sampled script request (500 tokens, card sampling) +back to back while a second process fills incompressible memory in 4 GB steps every 45 s up to +40 GB, holds it for 2 minutes, releases it and measures 2 minutes more. Server: the bundle +through `run_v41.sh`, 183 residents, drafter on. + +| Phase | tok/s, median (range), tau 0.3 | tok/s, median (range), tau 0.2 | +|---|---|---| +| Before | 36.7 (30.2–42.6) | 39.8 (37.1–42.4) | +| Taking 1–20 GB | 29.5 (25.9–32.2) | 31.4 (27.4–35.8) | +| Taking 21–40 GB | 32.8 (29.0–34.9) | 32.7 (20.7–35.4) | +| Holding 40 GB | 33.5 (30.5–34.5) | 32.2 (20.5–34.8) | +| After release | 35.9 (34.0–37.2) | 31.8 (23.5–34.6) | + +- The server keeps its 83 GiB in both runs and answers every request (61 and 62). macOS + compresses up to 24 GB and swaps 20 GB out (5 to 25 GB of swap); the pressure level reaches + "warning", never "critical". With 40 GB held, decoding is 9% slower at tau 0.3; the slowest + phase is the first allocation steps (20%), when the other process is filling memory. +- At tau 0.3 the cache installs 0.17 experts per token (1.5 MB read per token) and 90% of + demanded slots are resident; at tau 0.2, 3.7 installs and 33.6 MB per token. The heavier + reads make single requests drop to 20 tok/s under pressure and slow the recovery. +- The tau 0.2 run was a launcher bug: `run_v41.sh` did not set the admission threshold and + the code's default is 0.2, while every measurement here used 0.3. The launcher now sets 0.3. + The bundle checks under C4 above (38–40 tok/s) ran at 0.2. +- What this does not show: the server's own pages being taken. Its footprint counts + compressed and swapped pages too, so whether any of its 83 GiB left RAM is not measured; + the speed says little of it did. diff --git a/scripts/v41_build/d3_pressure.py b/scripts/v41_build/d3_pressure.py new file mode 100644 index 0000000..d7ab7ec --- /dev/null +++ b/scripts/v41_build/d3_pressure.py @@ -0,0 +1,95 @@ +"""D3: decode speed of a running server while another process takes memory. + +Sends one request after another to an OpenAI-compatible endpoint. After a baseline it allocates +incompressible memory in steps, holds it, lets it go and keeps measuring. Stepping stops early at +a critical pressure level or at ``--max-swap-gb`` of swap. Writes ``requests.jsonl`` (one line per +request) and ``samples.jsonl`` (every 5 s: memory taken here, swap, pageouts, compressor, free +pages, pressure level, the server's footprint) into ``--out``. Standard library only. + + python d3_pressure.py --pid SERVER_PID --request REQ.json --out DIR +""" + +import argparse +import json +import os +import re +import subprocess +import threading +import time +import urllib.request +from pathlib import Path + + +def sh(*cmd: str) -> str: + return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout + + +def system(pid: int) -> dict: + vm = {k.strip('" '): int(v) for k, v in re.findall(r'^(.+?):\s+(\d+)\.$', sh("vm_stat"), re.MULTILINE)} + page = int(sh("sysctl", "-n", "hw.pagesize")) + swap = float(re.search(r"used = ([\d.]+)M", sh("sysctl", "-n", "vm.swapusage"))[1]) / 1024 + return {"level": int(sh("sysctl", "-n", "kern.memorystatus_vm_pressure_level")), "swap_gb": round(swap, 2), + "free_gb": round(vm["Pages free"] * page / 2**30, 1), + "compressor_gb": round(vm["Pages occupied by compressor"] * page / 2**30, 1), + "pageouts": vm["Pageouts"], "swapouts": vm["Swapouts"], + "server": sh("top", "-l", "1", "-pid", str(pid), "-stats", "mem").split()[-1]} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--pid", type=int, required=True) + p.add_argument("--request", type=Path, required=True) + p.add_argument("--out", type=Path, required=True) + p.add_argument("--url", default="http://127.0.0.1:8017/v1/chat/completions") + p.add_argument("--step-gb", type=int, default=4) + p.add_argument("--max-gb", type=int, default=40) + p.add_argument("--max-swap-gb", type=float, default=24) + p.add_argument("--every", type=int, default=45, help="seconds per step") + p.add_argument("--rest", type=int, default=120, help="seconds of baseline, of hold and of recovery") + args = p.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + body, t0, state, held = args.request.read_bytes(), time.time(), {"phase": "baseline", "run": True}, [] + + def requests() -> None: + with (args.out / "requests.jsonl").open("w") as f: + while state["run"]: + start, phase, taken = time.time() - t0, state["phase"], len(held) + req = urllib.request.Request(args.url, body, {"Content-Type": "application/json"}) + try: + usage = json.load(urllib.request.urlopen(req, timeout=1800))["usage"] + except Exception as e: # noqa: BLE001 (a dead server is a result) + usage = {"error": f"{type(e).__name__}: {e}"[:200]} + time.sleep(5) + f.write(json.dumps({"t": round(start, 1), "end": round(time.time() - t0, 1), "phase": phase, "taken_gb": taken, + **{k: usage.get(k) for k in ("completion_tokens", "generation_tokens_per_second", + "time_to_first_token", "error") if k in usage}}) + "\n") + f.flush() + + def samples() -> None: + with (args.out / "samples.jsonl").open("w") as f: + while state["run"]: + state["last"] = system(args.pid) + f.write(json.dumps({"t": round(time.time() - t0, 1), "phase": state["phase"], "taken_gb": len(held), + **state["last"]}) + "\n") + f.flush() + time.sleep(5) + + threads = [threading.Thread(target=fn, daemon=True) for fn in (requests, samples)] + for t in threads: + t.start() + time.sleep(args.rest) + state["phase"] = "step" + while len(held) < args.max_gb and state["last"]["level"] < 4 and state["last"]["swap_gb"] < args.max_swap_gb: + held.extend(os.urandom(1 << 30) for _ in range(args.step_gb)) # random: the compressor gains nothing + time.sleep(args.every) + state["phase"] = "hold" + time.sleep(args.rest) + held.clear() + state["phase"] = "recovery" + time.sleep(args.rest) + state["run"] = False + threads[0].join(timeout=300) + + +if __name__ == "__main__": + main() diff --git a/scripts/v41_build/run_v41.sh b/scripts/v41_build/run_v41.sh index 276404e..473e0f4 100755 --- a/scripts/v41_build/run_v41.sh +++ b/scripts/v41_build/run_v41.sh @@ -30,6 +30,7 @@ if [ ! -x "$OMLX/.venv/bin/python" ]; then fi echo "endpoint http://127.0.0.1:$PORT/v1, model v41-lattice" >&2 cd "$OMLX" -OMLX_COMPACT=$B/models/v41-lattice/compact-map.json OMLX_LATTICE=$B/pool.json \ +# tau 0.3 is the admission threshold every measurement in the build plan used (the code's default is 0.2) +OMLX_COMPACT=$B/models/v41-lattice/compact-map.json OMLX_LATTICE=$B/pool.json OMLX_ROUTED_TAU=${OMLX_ROUTED_TAU:-0.3} \ exec .venv/bin/python "$REPO/scripts/v41_build/omlx_lattice.py" serve --model-dir "$B/models" --base-path "$B/base" \ --port "$PORT" --memory-guard-gb $((RAM * 3 / 4)) --max-concurrent-requests 1 --no-cache --headless