From 73c9471b0b4ce8f053e8b60812f5aeddcc86016c Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:30:10 +0800 Subject: [PATCH 1/2] ci: stage hosted AI research runner migration Co-Authored-By: Codex --- .../workflows/engineering-material-review.yml | 44 +++++++++++++++++- docs/hosted-ai-runners.md | 46 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 docs/hosted-ai-runners.md diff --git a/.github/workflows/engineering-material-review.yml b/.github/workflows/engineering-material-review.yml index 97ef9ebf..a1567480 100644 --- a/.github/workflows/engineering-material-review.yml +++ b/.github/workflows/engineering-material-review.yml @@ -9,8 +9,8 @@ concurrency: cancel-in-progress: false jobs: review: - if: github.ref == 'refs/heads/main' && github.run_attempt == 1 && vars.AI_SERVICE_RELEASE_READY == 'true' - runs-on: [self-hosted, research-runner] + if: github.ref == 'refs/heads/main' && github.run_attempt == 1 && vars.AI_SERVICE_RELEASE_READY == 'true' && vars.HOSTED_RESEARCH_ACCEPTED == 'true' + runs-on: ubuntu-latest timeout-minutes: 35 env: AI_SERVICE_URL: ${{ vars.AI_SERVICE_URL }} @@ -19,6 +19,46 @@ jobs: AUDIT_MATERIAL: ${{ vars.AI_SERVICE_ENGINEERING_MATERIAL_PATH }} AUDIT_ROLES: ${{ vars.AI_SERVICE_ENGINEERING_ROLES_JSON }} steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - name: Stage the approved hosted input bundle + env: + GH_TOKEN: ${{ github.token }} + HOSTED_INPUT_RELEASE: ${{ vars.HOSTED_INPUT_RELEASE }} + HOSTED_INPUT_ASSET: ${{ vars.HOSTED_INPUT_ASSET }} + HOSTED_INPUT_SHA256: ${{ vars.HOSTED_INPUT_SHA256 }} + run: | + set -euo pipefail + test -n "$HOSTED_INPUT_RELEASE" && test -n "$HOSTED_INPUT_ASSET" + [[ "$HOSTED_INPUT_SHA256" =~ ^[0-9a-f]{64}$ ]] + [[ "$HOSTED_INPUT_ASSET" =~ ^[A-Za-z0-9_.-]+\.zip$ ]] + gh release download "$HOSTED_INPUT_RELEASE" --repo "$GITHUB_REPOSITORY" --pattern "$HOSTED_INPUT_ASSET" --dir "$RUNNER_TEMP/approved-input" + python - <<'PYCODE' + import hashlib, os, stat, zipfile + from pathlib import Path, PurePosixPath + archive = Path(os.environ['RUNNER_TEMP']) / 'approved-input' / os.environ['HOSTED_INPUT_ASSET'] + if hashlib.sha256(archive.read_bytes()).hexdigest() != os.environ['HOSTED_INPUT_SHA256']: + raise SystemExit('approved hosted input digest mismatch') + target = Path('hosted-inputs') + with zipfile.ZipFile(archive) as bundle: + entries = bundle.infolist() + if len(entries) > 100000 or sum(item.file_size for item in entries) > 8 * 1024**3: + raise SystemExit('hosted input bundle exceeds the staging limit') + for item in entries: + path = PurePosixPath(item.filename) + if path.is_absolute() or '..' in path.parts or '\' in item.filename or stat.S_ISLNK(item.external_attr >> 16): + raise SystemExit('unsafe hosted input bundle entry') + bundle.extractall(target) + PYCODE + - name: Install the checked-out caller + run: | + python -m pip install . + python -m pip install --no-index --find-links hosted-inputs/wheels -r hosted-inputs/runtime-requirements.txt + python -m pip check - name: Review caller-owned frozen material using the approved runtime run: | set -euo pipefail diff --git a/docs/hosted-ai-runners.md b/docs/hosted-ai-runners.md new file mode 100644 index 00000000..527466f8 --- /dev/null +++ b/docs/hosted-ai-runners.md @@ -0,0 +1,46 @@ +# Hosted research/review runner migration (P1) + +The selected workflows use fresh GitHub-hosted Linux runners. Their existing +OIDC admission, RELEASE_READY/run-attempt guards, manual execute switches, +concurrency groups, task timeouts and research limits are unchanged. This PR +has no auth grant, broker/trading configuration, deployment or IAM changes. + +## Inputs and first execution + +A self-hosted installation and its absolute paths are not available. Stage a +reviewed ZIP release asset from **this repository** using existing contents-read +GITHUB_TOKEN: HOSTED_INPUT_RELEASE, HOSTED_INPUT_ASSET, HOSTED_INPUT_SHA256. +The SHA256 is mandatory; traversal/symlinks and excessive expanded data fail. +Publish no credentials, .env files, CLI login data or credential-bearing git +config. Existing per-workflow material/source/wheel/root variables must point +inside `hosted-inputs/` (output roots can use RUNNER_TEMP). Frozen source git +metadata/commit pins and the existing wheel SHA256 checks still apply; do not +replace a reviewed wheel with a similarly named package from an index. +The bundle includes only licensed, approved research inputs and required +reviewed runtime wheels. Publishing private inputs requires owner approval. + +The bundle must supply wheels/ and a fully pinned runtime-requirements.txt +for the approved task-client distribution and dependencies. Offline pip uses +only those reviewed artifacts; it cannot substitute an index package. The +caller itself is installed from this checkout. No client version/grant/model +is guessed by this PR. HOSTED_RESEARCH_ACCEPTED defaults to disabled; set it +only after input provenance, dependency pins and durable resume are accepted. +UES retains scripts/install_ai_task_client.py's exact approved wheel checks. +Missing bundles/wheels/relative paths/grants fail; no VPS fallback is provided. +Do not enable RELEASE_READY to bypass those prerequisites. + +Hosted disks are ephemeral. Pending task/result/learning ledgers must be +exported as approved private artifacts before job termination, and a resumed +run must consume that exact latest ledger. A workflow rerun with an old bundle +is not a safe resume. Do not execute live research until persistence/resume and +licensed input accessibility have been accepted; retain this PR as draft +until those conditions are resolved. CI verifies engineering syntax/tests; +it does not call Dot or establish research/receipt completion. + +## Acceptance and rollback + +Run actionlint and the existing PR CI at the final commit. Before adoption, +execute a synthetic/offline bundle and confirm wheel/input digest rejection, +frozen-source identity, existing budgets and Docker availability where needed. +No production scheduler/trading changes are in this PR. Revert through a new +PR if needed; re-enabling a VPS runner needs separate owner approval. From 6e02fd13961a15081c82ec673b15729bf62700c7 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:49:41 +0800 Subject: [PATCH 2/2] fix: verify hosted input staging with offline bundles Co-Authored-By: Codex --- .../workflows/engineering-material-review.yml | 6 ++- .github/workflows/hosted-input-ci.yml | 20 +++++++ scripts/check_hosted_input_bundle.py | 54 +++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/hosted-input-ci.yml create mode 100644 scripts/check_hosted_input_bundle.py diff --git a/.github/workflows/engineering-material-review.yml b/.github/workflows/engineering-material-review.yml index a1567480..e6f6da3f 100644 --- a/.github/workflows/engineering-material-review.yml +++ b/.github/workflows/engineering-material-review.yml @@ -41,7 +41,9 @@ jobs: import hashlib, os, stat, zipfile from pathlib import Path, PurePosixPath archive = Path(os.environ['RUNNER_TEMP']) / 'approved-input' / os.environ['HOSTED_INPUT_ASSET'] - if hashlib.sha256(archive.read_bytes()).hexdigest() != os.environ['HOSTED_INPUT_SHA256']: + with archive.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + if digest != os.environ['HOSTED_INPUT_SHA256']: raise SystemExit('approved hosted input digest mismatch') target = Path('hosted-inputs') with zipfile.ZipFile(archive) as bundle: @@ -50,7 +52,7 @@ jobs: raise SystemExit('hosted input bundle exceeds the staging limit') for item in entries: path = PurePosixPath(item.filename) - if path.is_absolute() or '..' in path.parts or '\' in item.filename or stat.S_ISLNK(item.external_attr >> 16): + if path.is_absolute() or '..' in path.parts or chr(92) in item.filename or stat.S_ISLNK(item.external_attr >> 16): raise SystemExit('unsafe hosted input bundle entry') bundle.extractall(target) PYCODE diff --git a/.github/workflows/hosted-input-ci.yml b/.github/workflows/hosted-input-ci.yml new file mode 100644 index 00000000..6176e5bb --- /dev/null +++ b/.github/workflows/hosted-input-ci.yml @@ -0,0 +1,20 @@ +name: Hosted input staging CI +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + stage: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - name: Execute staging code with offline synthetic inputs only + run: python scripts/check_hosted_input_bundle.py diff --git a/scripts/check_hosted_input_bundle.py b/scripts/check_hosted_input_bundle.py new file mode 100644 index 00000000..1d33fc7d --- /dev/null +++ b/scripts/check_hosted_input_bundle.py @@ -0,0 +1,54 @@ +"""Execute the actual embedded staging code on safe and hostile offline bundles.""" +import hashlib +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import zipfile + + +def main(): + checked = 0 + for workflow in sorted(Path('.github/workflows').glob('*.yml')): + text = workflow.read_text() + if "Stage the approved hosted input bundle" not in text: + continue + source = text.split(" python - <<'PYCODE'\n", 1)[1].split(' PYCODE', 1)[0] + source = '\n'.join(line[10:] for line in source.splitlines()) + compile(source, str(workflow), 'exec') + for kind in ('valid', 'digest', 'traversal', 'backslash', 'symlink'): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + destination = root / 'approved-input' + destination.mkdir() + archive = destination / 'fixture.zip' + member = zipfile.ZipInfo({ + 'traversal': '../escaped.txt', 'backslash': 'bad\\file', + }.get(kind, 'wheels/fixture.txt')) + if kind == 'symlink': + member.create_system = 3 + member.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(archive, 'w') as bundle: + bundle.writestr(member, 'offline synthetic input') + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + env = dict(os.environ, RUNNER_TEMP=str(root), HOSTED_INPUT_ASSET=archive.name, + HOSTED_INPUT_SHA256='0' * 64 if kind == 'digest' else digest) + result = subprocess.run([sys.executable, '-c', source], cwd=root, env=env, + capture_output=True, timeout=20, check=False) + if (result.returncode == 0) != (kind == 'valid'): + raise SystemExit(f'{workflow.name}: {kind} staging acceptance failed') + if kind == 'valid': + assert (root / 'hosted-inputs/wheels/fixture.txt').is_file() + else: + assert not (root / 'hosted-inputs').exists() + assert not (root / 'escaped.txt').exists() + checked += 1 + if not checked: + raise SystemExit('no hosted staging code was checked') + print(f'Hosted input staging: {checked} offline cases passed') + + +if __name__ == '__main__': + main()