diff --git a/src/client/Client.js b/src/client/Client.js index b1e6984..f934f90 100644 --- a/src/client/Client.js +++ b/src/client/Client.js @@ -11,6 +11,19 @@ let maxChunkCoord = 0xFFFFF let minPixelCoord = ~0xFFFFFF let maxPixelCoord = 0xFFFFFF +// Batched chunk request: [u16 guard][u16 count][u16 reserved] + count * (i32 x, i32 y). +// Length is always 6 + 8*count, which no other packet length can equal. +let chunkBatchGuard = 25565 +let maxChunkBatchCount = 4095 +let maxChunkBatchBytes = 512 * 1024 + +// Region LOD request: [u16 guard][u16 reserved][i32 rx][i32 ry][u16 w][u16 h] +let regionLodGuard = 25566 +let maxLodRegions = 1024 +let maxLodBytes = 256 * 1024 +let minRegionCoord = ~0xFFFF +let maxRegionCoord = 0xFFFF + let maxMessageLengths = [ 128, 128, @@ -365,6 +378,11 @@ export class Client { return } message = Buffer.from(message) + //batched chunk request, variable length so checked before the switch + if (message.length >= 14 && message.length % 8 === 6 && message.readUInt16LE(0) === chunkBatchGuard) { + this.handleChunkBatch(message) + return + } switch (message.length) { //request chunk case 8: { @@ -606,6 +624,20 @@ export class Client { this.y = y return } + //region level-of-detail request + case 16: { + if (message.readUInt16LE(0) !== regionLodGuard) { + this.destroy() + return + } + this.handleRegionLod( + message.readInt32LE(4), + message.readInt32LE(8), + message.readUInt16LE(12), + message.readUInt16LE(14) + ) + return + } //rank verification case 1: { if (message[0] > this.rank) { @@ -620,6 +652,127 @@ export class Client { } } + handleChunkBatch(message) { + let count = message.readUInt16LE(2) + if (count === 0 || count !== (message.length - 6) / 8 || count > maxChunkBatchCount) { + this.destroy() + return + } + let parts = [] + let pending = 3 + for (let i = 0; i < count; i++) { + let offset = 6 + i * 8 + let chunkX = message.readInt32LE(offset) + if (chunkX > maxChunkCoord || chunkX < minChunkCoord) { + this.destroy() + return + } + let chunkY = message.readInt32LE(offset + 4) + if (chunkY > maxChunkCoord || chunkY < minChunkCoord) { + this.destroy() + return + } + let chunkLocation = (chunkY & 0xf) << 4 | chunkX & 0xf + let regionId = ((chunkX >> 4) + 0x10000) + (((chunkY >> 4) + 0x10000) * 0x20000) + let region = this.world.getRegion(regionId) + if (!region.loaded) { + let deferredActions = this.handleUnloaded(region) + if (!deferredActions) return + let buffer = Buffer.allocUnsafe(1) + buffer[0] = chunkLocation + deferredActions.push(buffer) + if (++this.deferredAmount >= 100000 && this.rank < 3) { + this.destroy() + return + } + continue + } + region.lastHeld = this.server.currentTick + let data = region.getChunkData(chunkLocation) + if (parts.length && pending + 2 + data.length > maxChunkBatchBytes) { + this.sendChunkBatch(parts) + parts = [] + pending = 3 + } + parts.push(data) + pending += 2 + data.length + } + if (parts.length) this.sendChunkBatch(parts) + } + + handleRegionLod(regionX, regionY, width, height) { + if (width === 0 || height === 0) return + if (width * height > maxLodRegions) { + this.destroy() + return + } + let parts = [] + let pending = 3 + for (let dy = 0; dy < height; dy++) { + for (let dx = 0; dx < width; dx++) { + let rx = regionX + dx + let ry = regionY + dy + if (rx > maxRegionCoord || rx < minRegionCoord || ry > maxRegionCoord || ry < minRegionCoord) continue + let regionId = (rx + 0x10000) + ((ry + 0x10000) * 0x20000) + let region = this.world.getRegion(regionId) + if (!region.loaded) { + let deferredActions = this.handleUnloaded(region) + if (!deferredActions) return + deferredActions.push(Buffer.allocUnsafe(3)) + if (++this.deferredAmount >= 100000 && this.rank < 3) { + this.destroy() + return + } + continue + } + region.lastHeld = this.server.currentTick + parts.push(rx, ry, region.getLodData()) + pending += 776 + if (pending >= maxLodBytes) { + this.sendRegionLod(parts) + parts = [] + pending = 3 + } + } + } + if (parts.length) this.sendRegionLod(parts) + } + + //0x0C: [u8 opcode][u16 regionCount] + regionCount * ([i32 rx][i32 ry][768 bytes]) + sendRegionLod(parts) { + let count = parts.length / 3 + let out = Buffer.allocUnsafeSlow(3 + count * 776) + out[0] = 0x0C + out.writeUInt16LE(count, 1) + let offset = 3 + for (let i = 0; i < parts.length; i += 3) { + out.writeInt32LE(parts[i], offset) + out.writeInt32LE(parts[i + 1], offset + 4) + parts[i + 2].copy(out, offset + 8) + offset += 776 + } + this.ws.send(out.buffer, true) + } + + //0x0B: [u8 opcode][u16 chunkCount] + chunkCount * ([u16 byteLength][0x02 packet]) + sendChunkBatch(parts) { + let total = 3 + for (let i = 0; i < parts.length; i++) total += 2 + parts[i].length + //allocUnsafeSlow, same reason as Region.getChunkData + let out = Buffer.allocUnsafeSlow(total) + out[0] = 0x0B + out.writeUInt16LE(parts.length, 1) + let offset = 3 + for (let i = 0; i < parts.length; i++) { + let part = parts[i] + out.writeUInt16LE(part.length, offset) + offset += 2 + part.copy(out, offset) + offset += part.length + } + this.ws.send(out.buffer, true) + } + handleUnloaded(region) { if (!region.beganLoading) { if (this.rank < 3 && !this.regionloadquota.canSpend()) { @@ -646,34 +799,64 @@ export class Client { let deferredActions = this.deferredRegionActions.get(regionId) this.deferredAmount -= deferredActions.length this.deferredRegionActions.delete(regionId) + //consecutive chunk loads are batched, flushed before any other action so + //ordering relative to pastes/erases stays exact + let pendingChunks = [] + let pendingBytes = 3 + let flushChunks = () => { + if (pendingChunks.length === 0) return + if (pendingChunks.length === 1) { + this.ws.send(pendingChunks[0].buffer, true) + } else { + this.sendChunkBatch(pendingChunks) + } + pendingChunks = [] + pendingBytes = 3 + } for (let action of deferredActions) { switch (action.length) { //request chunk case 1: { - region.requestChunk(this, action[0]) + region.lastHeld = this.server.currentTick + let data = region.getChunkData(action[0]) + if (pendingChunks.length && pendingBytes + 2 + data.length > maxChunkBatchBytes) flushChunks() + pendingChunks.push(data) + pendingBytes += 2 + data.length + continue + } + //region level-of-detail + case 3: { + flushChunks() + region.lastHeld = this.server.currentTick + this.sendRegionLod([region.x, region.y, region.getLodData()]) continue } //set pixel case 5: { + flushChunks() region.setPixel(this, action[0], action[1], action[2], action[3], action[4]) continue } //chunk paste case 769: { + flushChunks() region.pasteChunk(action[0], action.subarray(1)) continue } //erase chunk case 4: { + flushChunks() region.eraseChunk(action[0], action[1], action[2], action[3]) continue } //protect chunk case 2: { + flushChunks() region.protectChunk(action[0], action[1]) } } } + flushChunks() } async handlePreWorld(message, isBinary) { diff --git a/src/region/Region.js b/src/region/Region.js index ece32a3..6d4de34 100644 --- a/src/region/Region.js +++ b/src/region/Region.js @@ -24,6 +24,7 @@ export class Region { this.lastHeld = this.server.currentTick this.loadPromise = null this.dataModified = false + this.lodCache = null this.destroyed = false } @@ -66,6 +67,38 @@ export class Region { this.isEmpty = false this.dataModified = true this.latestDataBuffer = null // allow gc of outdated db buffer + this.lodCache = null + } + + getLodData() { + if (this.lodCache) return this.lodCache + let out = Buffer.allocUnsafeSlow(768) + if (this.isEmpty === true || !this.pixels) { + let color = this.world.bgcolor.value + let r = color >> 16, g = (color & 0x00ff00) >> 8, b = color & 0x0000ff + for (let i = 0; i < 256; i++) { + out[i * 3] = r + out[i * 3 + 1] = g + out[i * 3 + 2] = b + } + this.lodCache = out + return out + } + for (let chunkLocation = 0; chunkLocation < 256; chunkLocation++) { + let base = chunkLocation * 768 + let r = 0, g = 0, b = 0 + for (let i = 0; i < 768; i += 3) { + r += this.pixels[base + i] + g += this.pixels[base + i + 1] + b += this.pixels[base + i + 2] + } + let o = chunkLocation * 3 + out[o] = (r / 256) | 0 + out[o + 1] = (g / 256) | 0 + out[o + 2] = (b / 256) | 0 + } + this.lodCache = out + return out } updateDataBuffer() { diff --git a/tools/fillWorld.js b/tools/fillWorld.js new file mode 100644 index 0000000..9752c05 --- /dev/null +++ b/tools/fillWorld.js @@ -0,0 +1,164 @@ +// Fills a world with random pixel data, writing regions straight into the LevelDB +// store the server uses. Intended for generating a large test world to benchmark +// chunk loading against. +// +// The server MUST be stopped while this runs - LevelDB holds an exclusive lock. +// +// Usage: +// node tools/fillWorld.js [--world main] [--radius 5000] [--mode noise|blocks] [--seed 1] +// +// --radius is in pixels from spawn in every direction, so the default 5000 covers a +// 10000x10000 area. Regions are 256x256 pixels and are the unit of storage, so the +// filled area is rounded outwards to whole regions. +// +// --mode noise : every pixel an independent random colour. Incompressible, so this is +// the worst case for both the on-disk encoding and the wire encoding. +// --mode blocks : random axis-aligned rectangles of flat colour. Compresses like a real +// world does, and is far smaller on disk. + +import { Level } from "level" +import { saveData } from "../src/region/regionData.js" + +const REGION_SIZE = 256 // pixels per region axis +const REGION_PIXEL_BYTES = 196608 // 256 * 256 * 3 + +function parseArgs(argv) { + const args = { world: "main", radius: 5000, mode: "noise", seed: 1 } + for (let i = 0; i < argv.length; i++) { + const key = argv[i] + if (!key.startsWith("--")) continue + const name = key.slice(2) + const value = argv[++i] + if (value === undefined) throw new Error(`missing value for --${name}`) + if (name === "world") args.world = value + else if (name === "radius") args.radius = parseInt(value, 10) + else if (name === "mode") args.mode = value + else if (name === "seed") args.seed = parseInt(value, 10) + else throw new Error(`unknown option --${name}`) + } + if (!Number.isInteger(args.radius) || args.radius <= 0) throw new Error("--radius must be a positive integer") + if (args.mode !== "noise" && args.mode !== "blocks") throw new Error("--mode must be 'noise' or 'blocks'") + return args +} + +// Deterministic PRNG so a given --seed reproduces the same world exactly. +function makeRandom(seed) { + let state = seed >>> 0 + if (state === 0) state = 0x9e3779b9 + return () => { + // xorshift32 + state ^= state << 13; state >>>= 0 + state ^= state >>> 17 + state ^= state << 5; state >>>= 0 + return state + } +} + +function fillNoise(pixels, rand) { + for (let i = 0; i < REGION_PIXEL_BYTES; i += 3) { + const r = rand() + pixels[i] = r & 0xff + pixels[i + 1] = (r >>> 8) & 0xff + pixels[i + 2] = (r >>> 16) & 0xff + } +} + +function fillBlocks(pixels, rand) { + // start from a flat background so untouched areas still compress well + const bg = rand() + pixels.fill(Buffer.from([bg & 0xff, (bg >>> 8) & 0xff, (bg >>> 16) & 0xff])) + const rectangles = 24 + (rand() % 40) + for (let n = 0; n < rectangles; n++) { + const c = rand() + const r = c & 0xff, g = (c >>> 8) & 0xff, b = (c >>> 16) & 0xff + const w = 4 + (rand() % 64) + const h = 4 + (rand() % 64) + const x0 = rand() % REGION_SIZE + const y0 = rand() % REGION_SIZE + const x1 = Math.min(REGION_SIZE, x0 + w) + const y1 = Math.min(REGION_SIZE, y0 + h) + for (let y = y0; y < y1; y++) { + let p = (y * REGION_SIZE + x0) * 3 + for (let x = x0; x < x1; x++) { + pixels[p++] = r + pixels[p++] = g + pixels[p++] = b + } + } + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)) + + // Region coordinates are pixel coordinates shifted right by 8, which floors correctly + // for negatives too, so this rounds the requested area outwards to whole regions. + const minRegion = (-args.radius) >> 8 + const maxRegion = (args.radius - 1) >> 8 + const perAxis = maxRegion - minRegion + 1 + const totalRegions = perAxis * perAxis + + console.log(`world : ${args.world}`) + console.log(`requested area : ${args.radius * 2} x ${args.radius * 2} pixels centred on spawn`) + console.log(`region range : ${minRegion}..${maxRegion} on both axes (${perAxis} x ${perAxis} = ${totalRegions} regions)`) + console.log(`covered area : ${perAxis * REGION_SIZE} x ${perAxis * REGION_SIZE} pixels`) + console.log(`mode : ${args.mode}`) + console.log(`seed : ${args.seed}`) + console.log("") + + const db = new Level("./data/regions", { keyEncoding: "utf8", valueEncoding: "buffer" }) + try { + await db.open() + } catch (err) { + console.error("Failed to open ./data/regions - is the server still running? LevelDB needs an exclusive lock.") + console.error(String(err.message ?? err)) + process.exit(1) + } + + const rand = makeRandom(args.seed) + const pixels = Buffer.allocUnsafe(REGION_PIXEL_BYTES) + const protection = Buffer.alloc(256) // 0 = unprotected + + const started = Date.now() + let written = 0 + let bytes = 0 + let batch = db.batch() + let batched = 0 + + for (let ry = minRegion; ry <= maxRegion; ry++) { + for (let rx = minRegion; rx <= maxRegion; rx++) { + if (args.mode === "noise") fillNoise(pixels, rand) + else fillBlocks(pixels, rand) + + const regionId = (rx + 0x10000) + ((ry + 0x10000) * 0x20000) + const data = saveData(protection, pixels) + batch.put(`${args.world}-${regionId}`, data) + bytes += data.length + batched++ + written++ + + if (batched >= 64) { + await batch.write() + batch = db.batch() + batched = 0 + const pct = ((written / totalRegions) * 100).toFixed(1) + const mb = (bytes / 1048576).toFixed(1) + process.stdout.write(`\r${written}/${totalRegions} regions (${pct}%), ${mb} MiB encoded`) + } + } + } + if (batched > 0) await batch.write() + + await db.close() + + const seconds = (Date.now() - started) / 1000 + console.log(`\r${written}/${totalRegions} regions (100.0%), ${(bytes / 1048576).toFixed(1)} MiB encoded`) + console.log("") + console.log(`Done in ${seconds.toFixed(1)}s.`) + console.log(`Wrote ${written} regions totalling ${(bytes / 1048576).toFixed(1)} MiB of encoded pixel data.`) +} + +main().catch(err => { + console.error(err) + process.exit(1) +})