Robj/cache invalid addresses

benchmarks/results-20261013/cache-invalid-addresses-main-7bb5fab/.gitattributes

@@ -0,0 +1,3 @@
+*.csv whitespace=cr-at-eol
+*.patch -whitespace
+*.log -whitespace

benchmarks/results-20261013/cache-invalid-addresses-main-7bb5fab/README.md

@@ -0,0 +1,11 @@
+# cache-invalid-addresses: main integration evidence
+
+Source merge: `f2dc7bf6dcd21214b03fd7e18ec91b940d50f30a`; main: `7bb5fab305b9c6179e31f75998dabea833537a08`.
+
+[Correctness checks](tests/README.md) include exact commands, hashes and retained initial failures. [Benchmark report](benchmarks/report.md), [project summary](benchmarks/summary.csv), and [verification status](benchmarks/status.json) describe the fresh real-disk screening.
+
+Resident single-thread point throughput lost 4.60% in the broad screen and 4.54% in the nearby warmed repeat, with non-overlapping observed sample ranges. The repeated eight-thread point result (-3.64%) overlaps main, and short ranges (+0.03%) do not reproduce the earlier loss. Short-run WAL inserts lost 3.00%. Control drift still limits precise attribution; performance neutrality is not established. The inherited async allocation stall documented above is a separate correctness/progress limitation.
+
+The campaign ran 133 comparison processes, 12 fixture creators and one recorded warmup through 11 exclusive jobs. These compare each updated feature with main; they do not isolate merge cost. Main controls drifted; sample-range overlap is descriptive, not a statistical confidence interval. Query/hash/barrier runs validate data, while insert-only runs check API returns, positive counts and successful close without readback. All benchmark databases were on ext4 `/dev/mapper/snafu-root_crypt` with O_DIRECT and fsynced fixture copies.
+
+`tests/archive-manifest.json` and `benchmarks/inventory.json` verify the small archived files. No database images, executables or core dumps are committed. Archived scripts retain original paths and must be adapted before replay outside the original campaign.

benchmarks/results-20261013/cache-invalid-addresses-main-7bb5fab/benchmarks/audit.json

@@ -0,0 +1,90 @@
+{
+  "complete": true,
+  "scope": "new main versus updated feature; merge cost not isolated",
+  "counts": {
+    "fixtures": {
+      "runs": 12,
+      "runner": "run_matrix-fixtures.py",
+      "verified": true
+    },
+    "resident": {
+      "runs": 23,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "disk": {
+      "runs": 23,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "prefix-resident-common": {
+      "runs": 4,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "prefix-resident-random": {
+      "runs": 4,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "prefix-disk-common": {
+      "runs": 4,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "prefix-disk-random": {
+      "runs": 4,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "insert": {
+      "runs": 31,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "hash": {
+      "runs": 4,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "barrier": {
+      "runs": 24,
+      "runner": "run_matrix.py",
+      "verified": true
+    },
+    "repeat-warmup": {
+      "runs": 1,
+      "runner": "repeat.py",
+      "verified": true
+    },
+    "repeat-cache-invalid-addresses": {
+      "runs": 4,
+      "runner": "repeat.py",
+      "verified": true
+    },
+    "repeat-directory-bucket-atomic": {
+      "runs": 4,
+      "runner": "repeat.py",
+      "verified": true
+    },
+    "repeat-async-durable-barrier": {
+      "runs": 4,
+      "runner": "repeat.py",
+      "verified": true
+    }
+  },
+  "total_processes": 146,
+  "comparison_processes": 133,
+  "fixture_processes": 12,
+  "recorded_warmup_processes": 1,
+  "exclusive_benchmark_jobs": 11,
+  "source_and_binary_hashes_verified": true,
+  "runner_hashes_verified": true,
+  "all_processes_serial": true,
+  "no_overlapping_build_test_jobs": true,
+  "resident_physical_read_bytes": 0,
+  "errors": 0,
+  "violations": 0,
+  "insert_verification": "API return, positive insert count and successful close; no readback",
+  "storage": "ext4 /dev/mapper/snafu-root_crypt; O_DIRECT; fsynced fixture copies"
+}

benchmarks/results-20261013/cache-invalid-addresses-main-7bb5fab/benchmarks/audit.py

@@ -0,0 +1,62 @@
+"""Validate final matrix coverage, provenance, row counts and exclusive scheduling."""
+import csv,hashlib,json
+from pathlib import Path
+C=Path(__file__).resolve().parent
+P=C.parents[2]
+O=P/'benchmarks/main-7bb5fab'
+def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
+def readcsv(p):return list(csv.DictReader(p.open()))
+expected={'fixtures':12,'resident':23,'disk':23,'prefix-resident-common':4,'prefix-resident-random':4,'prefix-disk-common':4,'prefix-disk-random':4,'insert':31,'hash':4,'barrier':24,'repeat-warmup':1,'repeat-cache-invalid-addresses':4,'repeat-directory-bucket-atomic':4,'repeat-async-durable-barrier':4}
+known_runner_hashes={sha(C/n):n for n in ['run_matrix.py','run_matrix-fixtures.py','repeat.py']}
+builds={p.name.removesuffix('-build.json'):json.loads(p.read_text()) for p in C.glob('*-build.json')}
+for profile,b in builds.items():
+ assert sha(Path(b['library']))==b['library_sha256'],profile
+ for f,h in b['source_files_sha256'].items():assert sha(Path(b['tree'])/f)==h,(profile,f)
+ for exe,v in b['binaries'].items():
+  assert sha(Path(v['binary']))==v['binary_sha256']
+  assert sha(Path(v['source']))==v['source_sha256']
+runs=[];counts={}
+for mode,count in expected.items():
+ folder=O/mode;m=json.loads((folder/'manifest.json').read_text())
+ assert m['complete'] and len(m['runs'])==count,mode
+ assert m['runner_sha256'] in known_runner_hashes,mode
+ if 'shared_runner_sha256' in m:assert m['shared_runner_sha256']==sha(C/'run_matrix.py')
+ for r in m['runs']:
+  assert r['verified'] and r['returncode']==0,(mode,r['label'])
+  log=folder/(r['label']+'.log');assert sha(log)==r['log_sha256']
+  assert 'FINDING: splinterdb_close' not in log.read_text(errors='replace')
+  b=builds[r['profile']];assert r['source_head']==b['head'] and r['library_sha256']==b['library_sha256']
+  assert r['binary_sha256'] in {v['binary_sha256'] for v in b['binaries'].values()}
+  assert r['finished']>=r['started'] and r['process_cpu_s']>0
+  runs.append(dict(mode=mode,label=r['label'],started=r['started'],finished=r['finished']))
+ counts[mode]=dict(runs=count,runner=known_runner_hashes[m['runner_sha256']],verified=True)
+ if (folder/'results.csv').exists():
+  rows=readcsv(folder/'results.csv')
+  if 'phase' in rows[0]:
+   for r in rows:
+    assert int(r['errors'])==0 and int(r['violations'])==0 and int(r['ops'])>0
+    if r['phase']=='warm':assert int(r['ops'])==int(r['n'])
+    elif mode=='resident' or mode.startswith('repeat-'):assert int(r['rd_bytes'])==0
+  if mode=='barrier':assert len(rows)==count and all(int(r['verified'])==1 for r in rows)
+  if mode=='insert':assert len(rows)==count and all(int(r['errors'])==0 and int(r['violations'])==0 and int(r['inserted'])>0 for r in rows)
+ if mode=='fixtures':
+  for r in m['runs']:
+   rows=readcsv(folder/(r['label']+'.csv'));assert len(rows)==1 and int(rows[0]['ops'])==int(rows[0]['n']) and int(rows[0]['errors'])==int(rows[0]['violations'])==0
+ if mode=='hash':
+  for r in m['runs']:assert [int(x['limit']) for x in r['rows']]==[1,2,10,100] and all(int(x['queries'])==1000 for x in r['rows'])
+  for i in range(4):assert len({r['rows'][i]['checksum'] for r in m['runs']})==1
+runs.sort(key=lambda r:r['started'])
+assert all(a['finished']<=b['started'] for a,b in zip(runs,runs[1:])), 'Benchmark subprocesses overlap'
+jobs=[json.loads(line) for line in (P/'coordination/jobs.jsonl').read_text().splitlines()]
+ours=[j for j in jobs if j['mode']=='benchmark' and j['name'].startswith('main-7bb5fab-benchmark-')]
+assert len(ours)==11 and all(j['mode']=='benchmark' and j['returncode']==0 for j in ours)
+for j in ours:
+ for k in jobs:
+  if j is k:continue
+  assert min(j['finished'],k['finished'])<=max(j['started'],k['started']),(j['name'],k['name'])
+for r in runs:assert sum(j['started']<=r['started'] and j['finished']>=r['finished'] for j in ours)==1
+(C/'run_job.py').write_bytes((P/'coordination/run_job.py').read_bytes())
+(C/'benchmark-jobs.json').write_text(json.dumps(ours,indent=2)+'\n')
+result=dict(complete=True,scope='new main versus updated feature; merge cost not isolated',counts=counts,total_processes=len(runs),comparison_processes=len(runs)-13,fixture_processes=12,recorded_warmup_processes=1,exclusive_benchmark_jobs=len(ours),source_and_binary_hashes_verified=True,runner_hashes_verified=True,all_processes_serial=True,no_overlapping_build_test_jobs=True,resident_physical_read_bytes=0,errors=0,violations=0,insert_verification='API return, positive insert count and successful close; no readback',storage='ext4 /dev/mapper/snafu-root_crypt; O_DIRECT; fsynced fixture copies')
+(C/'audit.json').write_text(json.dumps(result,indent=2)+'\n')
+print(json.dumps(result,indent=2))

benchmarks/results-20261013/cache-invalid-addresses-main-7bb5fab/benchmarks/benchmark-jobs.json

@@ -0,0 +1,167 @@
+[
+  {
+    "name": "main-7bb5fab-benchmark-fixtures",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "fixtures"
+    ],
+    "started": 1791905953.791713,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791905996.8644083
+  },
+  {
+    "name": "main-7bb5fab-benchmark-resident",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "resident"
+    ],
+    "started": 1791906087.9268048,
+    "pid": 5,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791906898.088736
+  },
+  {
+    "name": "main-7bb5fab-benchmark-disk",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "disk"
+    ],
+    "started": 1791906929.7655535,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791907361.8222167
+  },
+  {
+    "name": "main-7bb5fab-benchmark-prefix",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "prefix"
+    ],
+    "started": 1791907361.8244245,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791907746.3817344
+  },
+  {
+    "name": "main-7bb5fab-benchmark-insert",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "insert"
+    ],
+    "started": 1791907746.3838046,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791907918.6362858
+  },
+  {
+    "name": "main-7bb5fab-benchmark-hash",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "hash"
+    ],
+    "started": 1791907918.6382897,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791907922.146424
+  },
+  {
+    "name": "main-7bb5fab-benchmark-barrier",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/run_matrix.py",
+      "barrier"
+    ],
+    "started": 1791908012.5013723,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791908144.8841226
+  },
+  {
+    "name": "main-7bb5fab-benchmark-repeat-warmup",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/repeat.py",
+      "warmup"
+    ],
+    "started": 1791908144.8859186,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791908167.146525
+  },
+  {
+    "name": "main-7bb5fab-benchmark-repeat-invalid",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/repeat.py",
+      "cache-invalid-addresses"
+    ],
+    "started": 1791908236.5269575,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791908369.5229552
+  },
+  {
+    "name": "main-7bb5fab-benchmark-repeat-atomic",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/repeat.py",
+      "directory-bucket-atomic"
+    ],
+    "started": 1791908442.7619998,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791908531.8354392
+  },
+  {
+    "name": "main-7bb5fab-benchmark-repeat-async",
+    "mode": "benchmark",
+    "cwd": "/home/rob/projects/splinterdb",
+    "command": [
+      "python3",
+      ".cache/projects-20261011/coordination/main-7bb5fab/benchmarks/repeat.py",
+      "async-durable-barrier"
+    ],
+    "started": 1791908542.3013017,
+    "pid": 3,
+    "db_dir": "/home/rob/projects/splinterdb/.cache/projects-20261011/benchmarks/main-7bb5fab",
+    "returncode": 0,
+    "finished": 1791908631.2491326
+  }
+]

benchmarks/results-20261013/cache-invalid-addresses-main-7bb5fab/benchmarks/build_all.py

@@ -0,0 +1,44 @@
+import hashlib,json,subprocess,sys
+from pathlib import Path
+ROOT=Path('/home/rob/projects/splinterdb')
+CAMP=ROOT/'.cache/projects-20261011'
+OUT=Path(__file__).resolve().parent
+SOURCES=OUT/'sources'
+def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
+profiles={'main':(ROOT,'build-main-7bb5fab')}
+for name in ['disk-format','filtered-ranges','log-group-limit','branch-prefix','lazy-merge','async-durable-barrier','cache-invalid-addresses']:
+ profiles[name]=(CAMP/'worktrees'/name,'build-main-7bb5fab')
+for opt in ['direct','bucket','bucket-simd','bucket-atomic']:
+ profiles['directory-'+opt]=(CAMP/'worktrees/cache-directory',f'build-main-7bb5fab/{opt}')
+flags=['cc','-D_GNU_SOURCE','-ggdb','-Wall','-pthread','-Werror','-DXXH_STATIC_LINKING_ONLY','-fPIC','-DSPLINTERDB_PLATFORM_DIR=platform_linux','-O3','-ffast-math','-flto','-march=native']
+manifest={}
+for name,(tree,buildroot) in profiles.items():
+ if len(sys.argv)>1 and name not in sys.argv[1:]:continue
+ head=subprocess.check_output(['git','rev-parse','HEAD'],cwd=tree,text=True).strip()
+ subprocess.run(['git','merge-base','--is-ancestor','7bb5fab',head],cwd=tree,check=True)
+ lib=tree/buildroot/'release/lib/libsplinterdb.a'
+ header=(tree/'include/splinterdb/splinterdb.h').read_text()
+ record=dict(tree=str(tree),head=head,library=str(lib),library_sha256=sha(lib),binaries={},source_files_sha256={})
+ for filename in subprocess.check_output(['git','ls-files','src','include'],cwd=tree,text=True).splitlines():
+  if (tree/filename).is_file():record['source_files_sha256'][filename]=sha(tree/filename)
+ bins=['kvbench','insert_benchmark']
+ if name in ['main','branch-prefix']:bins+=['prefix_kvbench']
+ if name in ['main','filtered-ranges']:bins+=['hash_range_benchmark']
+ if name in ['main','async-durable-barrier']:bins+=['barrier_bench']
+ for binaryname in bins:
+  source=SOURCES/(binaryname+'.c');binary=OUT/'bin'/name/binaryname;binary.parent.mkdir(parents=True,exist_ok=True)
+  macros=[]
+  if binaryname in ['kvbench','prefix_kvbench']:macros+=['-DKVBENCH_CLOSE_HAS_FORCE']
+  if binaryname=='prefix_kvbench':
+   macros+=['-DKVBENCH_SPACE_MODE']
+   if name=='branch-prefix':macros+=['-DKVBENCH_PREFIX_SUPPORT']
+  if binaryname=='hash_range_benchmark' and name=='filtered-ranges':macros+=['-DHASH_RANGE_FILTER_API']
+  if binaryname=='insert_benchmark' and name!='log-group-limit':macros+=['-DKVBENCH_NO_LOG_GROUP_LIMIT']
+  if '#define SPLINTERDB_NOTIFICATION_BLOCKING' in header:macros+=['-DHAVE_SENTINEL_NOTIFICATION','-DHAVE_ASYNC_BARRIER']
+  command=flags+macros+['-I',str(tree/'include'),'-I',str(tree/'src'),'-I',str(tree/'src/platform_linux'),str(source),str(lib),'-laio','-lxxhash','-lm','-lpthread','-o',str(binary)]
+  print('build',name,binaryname,flush=True)
+  subprocess.run(command,check=True)
+  record['binaries'][binaryname]=dict(command=command,binary=str(binary),binary_sha256=sha(binary),source=str(source),source_sha256=sha(source))
+ (OUT/(name+'-build.json')).write_text(json.dumps(record,indent=2)+'\n')
+ manifest[name]=record
+(OUT/'builds-last.json').write_text(json.dumps(manifest,indent=2)+'\n')