diff --git a/deploy/docker-compose.validator.yml b/deploy/docker-compose.validator.yml index 389e4c9..fb844ad 100644 --- a/deploy/docker-compose.validator.yml +++ b/deploy/docker-compose.validator.yml @@ -17,6 +17,10 @@ services: MT_RELAY_URL: "${MT_RELAY_URL:-https://api.microtensor.cloud}" MT_GPUS: "${MT_GPUS:-0,1,2}" MT_SANDBOX_RUNTIME: "${MT_SANDBOX_RUNTIME:-runsc}" + MT_JUDGE_SHARE_GPU: "${MT_JUDGE_SHARE_GPU:-}" + MT_JUDGE_MODEL: "${MT_JUDGE_MODEL:-}" + MT_JUDGE_REVISION: "${MT_JUDGE_REVISION:-}" + MT_SLM_MEMORY_SHARE: "${MT_SLM_MEMORY_SHARE:-0}" HF_HOME: /var/lib/microtensor/cache/hf volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/deploy/validator-entrypoint.sh b/deploy/validator-entrypoint.sh index 3715b95..25a6107 100644 --- a/deploy/validator-entrypoint.sh +++ b/deploy/validator-entrypoint.sh @@ -6,8 +6,8 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi -if ! docker info --format '{{json .Runtimes}}' | grep -q runsc; then - echo "gVisor (runsc) is not registered with Docker; miner code will not run" >&2 +if [ "${MT_SANDBOX_RUNTIME:-runsc}" = "runsc" ] && ! docker info --format '{{json .Runtimes}}' | grep -q runsc; then + echo "gVisor (runsc) is not registered with Docker; set MT_SANDBOX_RUNTIME=runc to run miner code in plain containers" >&2 exit 1 fi diff --git a/docs/validator_setup.md b/docs/validator_setup.md index 1b2adea..7868063 100644 --- a/docs/validator_setup.md +++ b/docs/validator_setup.md @@ -116,6 +116,20 @@ docker compose -f docker-compose.validator.yml run --rm validator epoch run --ar Evaluates twenty items per test, posts no report and sets no weights. Use it after installation and after every image update. +## Testnet on one GPU + +The published mechanism gives the context compression probe agent its own card. On testnet a validator may run everything on one GPU and swap the probe agent for a quantised build of the same model so both fit: + +``` +MT_GPUS=0 +MT_SANDBOX_RUNTIME=runc +MT_JUDGE_SHARE_GPU=1 +MT_JUDGE_MODEL=Qwen/Qwen2.5-Coder-7B-Instruct-AWQ +MT_SLM_MEMORY_SHARE=0.38 +``` + +The probe agent then takes half the card and the model under test the share you set. `MT_JUDGE_MODEL` is refused on mainnet; `MT_SANDBOX_RUNTIME=runc` runs miner code in a plain container, which is acceptable only on testnet. + ## Updating ```bash diff --git a/microtensor_subnet/cli/epoch.py b/microtensor_subnet/cli/epoch.py index bb9e214..153064f 100644 --- a/microtensor_subnet/cli/epoch.py +++ b/microtensor_subnet/cli/epoch.py @@ -95,6 +95,7 @@ def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> ) run.add_argument("--concurrency", type=int, default=3) run.add_argument("--sandbox-runtime", default=os.environ.get("MT_SANDBOX_RUNTIME", "runsc")) + _add_gpu_sharing(run) run.add_argument("--item-limit", type=int, default=0, help="development only") run.add_argument("--dry-run", action="store_true", help="evaluate and report, set no weights") add_chain_arguments(run) @@ -110,6 +111,7 @@ def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> ) loop.add_argument("--concurrency", type=int, default=3) loop.add_argument("--sandbox-runtime", default=os.environ.get("MT_SANDBOX_RUNTIME", "runsc")) + _add_gpu_sharing(loop) loop.add_argument("--poll-seconds", type=int, default=600) loop.add_argument("--once", action="store_true", help="one poll, then exit") loop.add_argument("--item-limit", type=int, default=0, help="development only") @@ -488,6 +490,27 @@ def _probe(args: argparse.Namespace) -> int: close_reference(docker, reference) +def _add_gpu_sharing(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--judge-share-gpu", + action="store_true", + default=os.environ.get("MT_JUDGE_SHARE_GPU", "") == "1", + help="run the probe agent on the same GPU as the model under test", + ) + parser.add_argument( + "--judge-model", + default=os.environ.get("MT_JUDGE_MODEL", ""), + help="testnet only: a different probe agent repository", + ) + parser.add_argument("--judge-revision", default=os.environ.get("MT_JUDGE_REVISION", "")) + parser.add_argument( + "--slm-memory-share", + type=float, + default=float(os.environ.get("MT_SLM_MEMORY_SHARE", "0") or 0), + help="vLLM memory share for the model under test; 0 picks the default", + ) + + def _settings(args: argparse.Namespace, config: Any) -> EpochConfig: return EpochConfig( chain=config, @@ -500,6 +523,10 @@ def _settings(args: argparse.Namespace, config: Any) -> EpochConfig: sandbox_runtime=args.sandbox_runtime, dry_run=args.dry_run, item_limit=args.item_limit, + judge_share_gpu=bool(getattr(args, "judge_share_gpu", False)), + judge_model=str(getattr(args, "judge_model", "") or ""), + judge_revision=str(getattr(args, "judge_revision", "") or ""), + slm_memory_share=float(getattr(args, "slm_memory_share", 0.0) or 0.0), ) diff --git a/microtensor_subnet/serving/host/config.py b/microtensor_subnet/serving/host/config.py index d7d7057..d989d2f 100644 --- a/microtensor_subnet/serving/host/config.py +++ b/microtensor_subnet/serving/host/config.py @@ -32,6 +32,7 @@ class HostConfig: sandbox_image: str = SANDBOX_IMAGE gateway_image: str = GATEWAY_IMAGE sandbox_runtime: str = "runsc" + slm_memory_share: float = 0.0 sandbox_cpus: float = SANDBOX_CPUS sandbox_memory: str = SANDBOX_MEMORY sandbox_pids: int = SANDBOX_PIDS diff --git a/microtensor_subnet/validator/config.py b/microtensor_subnet/validator/config.py index f0ac38b..ee5c80e 100644 --- a/microtensor_subnet/validator/config.py +++ b/microtensor_subnet/validator/config.py @@ -42,6 +42,10 @@ class EpochConfig: slm_ready_seconds: int = SLM_READY_SECONDS host_services_port: int = 9701 allow_unsandboxed: bool = False + judge_share_gpu: bool = False + judge_model: str = "" + judge_revision: str = "" + slm_memory_share: float = 0.0 dry_run: bool = False item_limit: int = 0 extra: dict[str, str] = field(default_factory=dict) diff --git a/microtensor_subnet/validator/epoch.py b/microtensor_subnet/validator/epoch.py index fc5624a..c72463a 100644 --- a/microtensor_subnet/validator/epoch.py +++ b/microtensor_subnet/validator/epoch.py @@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import Any, Final from microtensor_subnet.artifact.params import ParameterError, footprint_bytes from microtensor_subnet.chain.epochs import Epoch, epoch_seed @@ -43,9 +43,10 @@ from microtensor_subnet.validator.sandbox import tools as tool_faults from microtensor_subnet.validator.sandbox.docker import Docker from microtensor_subnet.validator.sandbox.services import Services -from microtensor_subnet.validator.sandbox.slm import SlmClient +from microtensor_subnet.validator.sandbox.slm import MEMORY_SHARE, SlmClient log = logging.getLogger("microtensor_subnet.validator.epoch") +JUDGE_SHARED_SHARE: Final[float] = 0.5 DEFAULT_CONTEXT = 8192 DEFAULT_MEMORY_BYTES = 24 * 1024**3 DEFAULT_MAX_PARAMETERS = 8_000_000_000 @@ -587,9 +588,12 @@ def run_epoch( gpus = tuple(config.gpus or ("0",)) agent: probe_judge.ProbeAgent | None = None if context.grader == compress.GRADER: - if len(gpus) < 2: - raise EpochError("context compression needs a second GPU for the probe agent") - agent, gpus = _judge(config, context, engine, gpus[-1]), gpus[:-1] + if config.judge_share_gpu: + agent = _judge(config, context, engine, gpus[-1], memory_share=JUDGE_SHARED_SHARE) + else: + if len(gpus) < 2: + raise EpochError("context compression needs a second GPU for the probe agent") + agent, gpus = _judge(config, context, engine, gpus[-1]), gpus[:-1] services.start() host = SandboxHost(config, engine, services, host_ip()) runner = Runner(config, context, host, relay, GpuPool(gpus), desk) @@ -721,7 +725,12 @@ def _owner(chain: Any) -> str: def _judge( - config: EpochConfig, context: EpochContext, engine: Docker, gpu: str + config: EpochConfig, + context: EpochContext, + engine: Docker, + gpu: str, + *, + memory_share: float = MEMORY_SHARE, ) -> probe_judge.ProbeAgent: grading = context.grading spec = grading.get("probe_agent") or {} @@ -729,6 +738,11 @@ def _judge( if not model: raise EpochError("the arena names no probe agent") revision = str(spec.get("revision") or "main") + if config.judge_model: + if config.chain.network != "test": + raise EpochError("a probe agent override is only allowed on testnet") + log.warning("testnet probe agent override: %s instead of %s", config.judge_model, model) + model, revision = config.judge_model, config.judge_revision or "main" model_dir = collect.download(model, revision, config.cache_dir) tokenizer = str(grading.get("tokenizer") or model) tokenizer_dir = ( @@ -740,6 +754,7 @@ def _judge( model_dir, gpu, max_tokens=int(spec.get("max_tokens") or 64), + memory_share=memory_share, ) agent.start(config.slm_ready_seconds) context.judge = compress.Judge( diff --git a/microtensor_subnet/validator/sandbox/__init__.py b/microtensor_subnet/validator/sandbox/__init__.py index d175a16..8268e93 100644 --- a/microtensor_subnet/validator/sandbox/__init__.py +++ b/microtensor_subnet/validator/sandbox/__init__.py @@ -59,6 +59,9 @@ def gateway_image(self) -> str: ... @property def sandbox_runtime(self) -> str: ... + @property + def slm_memory_share(self) -> float: ... + @property def sandbox_cpus(self) -> float: ... @@ -206,7 +209,9 @@ def _slm_container( cpu_container(name, self.config.llama_image, model_dir, max_len, limits, network) ) return name - share = SHARED_MEMORY_SHARE if limits.router_on_gpu else MEMORY_SHARE + share = self.config.slm_memory_share or ( + SHARED_MEMORY_SHARE if limits.router_on_gpu else MEMORY_SHARE + ) self.docker.start( ContainerSpec( name=name, diff --git a/microtensor_subnet/validator/sandbox/judge.py b/microtensor_subnet/validator/sandbox/judge.py index 4780fb0..978b006 100644 --- a/microtensor_subnet/validator/sandbox/judge.py +++ b/microtensor_subnet/validator/sandbox/judge.py @@ -11,7 +11,12 @@ import httpx from microtensor_subnet.validator.sandbox.docker import ContainerSpec, Docker -from microtensor_subnet.validator.sandbox.slm import MODEL_NAME, SlmClient, vllm_command +from microtensor_subnet.validator.sandbox.slm import ( + MEMORY_SHARE, + MODEL_NAME, + SlmClient, + vllm_command, +) PROMPT: Final[str] = ( "You read a summary of a coding agent's session and answer one question about that session " @@ -36,6 +41,7 @@ class ProbeAgent: model_dir: Path gpu: str max_tokens: int = 64 + memory_share: float = MEMORY_SHARE name: str = "" network: str = "" url: str = "" @@ -50,7 +56,9 @@ def start(self, ready_seconds: int) -> None: name=self.name, image=self.image, network=self.network, - command=vllm_command("/model", max_model_len=MAX_MODEL_LEN), + command=vllm_command( + "/model", max_model_len=MAX_MODEL_LEN, memory_share=self.memory_share + ), env={"HF_HUB_OFFLINE": "1", "VLLM_NO_USAGE_STATS": "1"}, volumes=((str(self.model_dir), "/model", "ro"),), gpus=(self.gpu,),