Keep dependency files local and persist Modal archives

.github/workflows/volume-canary.yml

@@ -20,19 +20,20 @@ jobs:
     runs-on: ["composal-x64-${{ inputs.size || '4' }}x"]
     steps:
       - uses: actions/checkout@v4
-      - uses: composalai/disk-volume@v1
+      - run: node --test post.test.cjs
+      - uses: ./
         id: disk
         with:
           key: volume-canary-v1
           path: /tmp/dependency-store
       - name: Verify restore and isolation
         env:
           CACHE_HIT: ${{ steps.disk.outputs.cache-hit }}
-          EXPECTED_HIT: ${{ inputs.expected_hit || 'true' }}
+          EXPECTED_HIT: ${{ inputs.expected_hit || '' }}
           EVENT: ${{ github.event_name }}
         run: |
           set -eu
-          test "$CACHE_HIT" = "$EXPECTED_HIT"
+          if [ -n "$EXPECTED_HIT" ]; then test "$CACHE_HIT" = "$EXPECTED_HIT"; fi
           findmnt -T /tmp/dependency-store
           if [ "$CACHE_HIT" = true ]; then
             test "$(cat /tmp/dependency-store/proof)" = trusted

README.md

@@ -26,9 +26,10 @@ keys for package download stores (`~/.npm`, a pnpm store, Cargo registry/git
 caches); package managers check the package versions and integrity. Use distinct
 keys for incompatible platforms or toolchains.
 
-Each VM starts with a private writable clone of trusted snapshots. Concurrent
-jobs never share a live directory. Changes are flushed in the action's post
-step. After the VM stops, Composal verifies the job's actual GitHub assignment:
+Each VM extracts trusted snapshots into private directories on its local disk,
+so package managers and tests read local files. Concurrent jobs never share a
+live directory. The action's post step archives its directory to the job's
+private Modal Volume namespace. After the VM stops, Composal verifies the job's actual GitHub assignment:
 only successful `push`, `schedule`, or `workflow_dispatch` jobs on the
 repository's default branch can publish a new snapshot. Pull requests and other
 branches can restore and modify their private clone, but their writes are
@@ -39,7 +40,7 @@ No Modal credentials or additional GitHub workflow permissions are needed.
 This action requires Linux Composal runners and host jobs. Container jobs are
 not supported in v1. The path must be empty and must not traverse a symlink;
 existing files are never hidden or replaced. Do not use this action to mount
-Docker's overlay storage; this is a distributed filesystem directory, not an
+Docker's overlay storage; this is a persisted dependency directory, not an
 ext4 block device. Keep deployment artifacts and irreplaceable data elsewhere.
 
 Limits: five disk mounts per job, 32 keys and 10 GiB of published data per

main.cjs

@@ -27,7 +27,7 @@ function mount(env = process.env) {
   if (fs.realpathSync(target) !== target || fs.readdirSync(target).length) throw new Error('Mount path must be an empty directory without symlinks');
   execFileSync('mount', ['--bind', source, target], {stdio:'inherit'});
   fs.writeFileSync(stateFile, JSON.stringify({key}), {flag:'wx'});
-  fs.appendFileSync(env.GITHUB_STATE, `target=${target}\nroot=${root}\n`);
+  fs.appendFileSync(env.GITHUB_STATE, `target=${target}\nroot=${root}\ndigest=${digest}\n`);
   fs.appendFileSync(env.GITHUB_OUTPUT, `cache-hit=${hit}\npath=${target}\n`);
   console.log(`Mounted ${key}: ${hit ? 'trusted snapshot restored' : 'empty volume'} at ${target}`);
 }

post.cjs

@@ -1,9 +1,31 @@
 'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
 const {execFileSync} = require('node:child_process');
-try {
-  if (process.env.STATE_target) {
-    execFileSync('sync', [process.env.STATE_root], {stdio:'inherit'});
-    execFileSync('umount', [process.env.STATE_target], {stdio:'inherit'});
-    console.log('Disk writes persisted to the job clone. Composal publishes successful default-branch jobs after cleanup.');
+
+function persist(env = process.env, run = execFileSync) {
+  if (!env.STATE_target) return;
+  if (!env.COMPOSAL_DISK_PERSIST_ROOT) run('sync', [env.STATE_root], {stdio:'inherit'});
+  run('umount', [env.STATE_target], {stdio:'inherit'});
+  if (env.COMPOSAL_DISK_PERSIST_ROOT) {
+    const digest = env.STATE_digest;
+    if (!/^[a-f0-9]{64}$/.test(digest || '')) throw new Error('Invalid disk identity');
+    const root = fs.realpathSync(env.STATE_root);
+    const backing = fs.realpathSync(env.COMPOSAL_DISK_PERSIST_ROOT);
+    const source = path.join(root, 'data', digest);
+    if (fs.realpathSync(source) !== source) throw new Error('Disk source must not be a symbolic link');
+    const archive = path.join(backing, 'archives', digest + '.tar.gz');
+    fs.mkdirSync(path.dirname(archive), {recursive:true});
+    fs.mkdirSync(path.join(backing, 'manifests'), {recursive:true});
+    run('tar', ['-czf', archive + '.tmp', '-C', source, '.'], {stdio:'inherit'});
+    fs.renameSync(archive + '.tmp', archive);
+    fs.copyFileSync(path.join(root, 'manifests', digest + '.json'), path.join(backing, 'manifests', digest + '.json'));
+    run('sync', [backing], {stdio:'inherit'});
+    console.log('Dependency snapshot archived to the private Modal disk.');
   }
-} catch (error) { console.error('Could not flush or unmount the disk: ' + error.message); process.exitCode = 1; }
+  console.log('Disk writes persisted to the job clone. Composal publishes successful default-branch jobs after cleanup.');
+}
+if (require.main === module) {
+  try { persist(); } catch (error) { console.error('Could not persist the disk: ' + error.message); process.exitCode = 1; }
+}
+module.exports = {persist};

post.test.cjs

@@ -0,0 +1,43 @@
+'use strict';
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const {execFileSync} = require('node:child_process');
+const {persist} = require('./post.cjs');
+
+test('post step preserves dependency bytes in the Modal archive', () => {
+  const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'composal-disk-post-'));
+  try {
+    const root = path.join(temp, 'working');
+    const backing = path.join(temp, 'persisted');
+    const digest = crypto.createHash('sha256').update('test-key').digest('hex');
+    const source = path.join(root, 'data', digest);
+    fs.mkdirSync(source, {recursive:true});
+    fs.mkdirSync(path.join(root, 'manifests'), {recursive:true});
+    fs.mkdirSync(backing);
+    fs.writeFileSync(path.join(source, 'dependency.txt'), 'cached dependency bytes');
+    fs.writeFileSync(path.join(root, 'manifests', digest + '.json'), JSON.stringify({key:'test-key'}));
+    persist({STATE_target:source, STATE_root:root, STATE_digest:digest, COMPOSAL_DISK_PERSIST_ROOT:backing}, (command, args, options) => {
+      if (command === 'tar') execFileSync(command, args, options);
+    });
+    const restored = path.join(temp, 'restored');
+    fs.mkdirSync(restored);
+    execFileSync('tar', ['-xzf', path.join(backing, 'archives', digest + '.tar.gz'), '-C', restored]);
+    assert.equal(fs.readFileSync(path.join(restored, 'dependency.txt'), 'utf8'), 'cached dependency bytes');
+    assert.deepEqual(JSON.parse(fs.readFileSync(path.join(backing, 'manifests', digest + '.json'))), {key:'test-key'});
+    assert.equal(fs.existsSync(path.join(backing, 'archives', digest + '.tar.gz.tmp')), false);
+  } finally { fs.rmSync(temp, {recursive:true, force:true}); }
+});
+
+test('original runner disks still flush before unmounting', () => {
+  const calls = [];
+  persist({STATE_target:'/target',STATE_root:'/disk'}, (...args) => calls.push(args.slice(0,2)));
+  assert.deepEqual(calls, [['sync',['/disk']],['umount',['/target']]]);
+});
+
+test('unmounted actions do not attempt publication', () => {
+  persist({}, () => assert.fail('unmounted action must not run commands'));
+});