Batch chunk requests and add region level-of-detail

src/client/Client.js

@@ -11,6 +11,19 @@ let maxChunkCoord = 0xFFFFF
 let minPixelCoord = ~0xFFFFFF
 let maxPixelCoord = 0xFFFFFF
 
+// Batched chunk request: [u16 guard][u16 count][u16 reserved] + count * (i32 x, i32 y).
+// Length is always 6 + 8*count, which no other packet length can equal.
+let chunkBatchGuard = 25565
+let maxChunkBatchCount = 4095
+let maxChunkBatchBytes = 512 * 1024
+
+// Region LOD request: [u16 guard][u16 reserved][i32 rx][i32 ry][u16 w][u16 h]
+let regionLodGuard = 25566
+let maxLodRegions = 1024
+let maxLodBytes = 256 * 1024
+let minRegionCoord = ~0xFFFF
+let maxRegionCoord = 0xFFFF
+
 let maxMessageLengths = [
 	128,
 	128,
@@ -365,6 +378,11 @@ export class Client {
 			return
 		}
 		message = Buffer.from(message)
+		//batched chunk request, variable length so checked before the switch
+		if (message.length >= 14 && message.length % 8 === 6 && message.readUInt16LE(0) === chunkBatchGuard) {
+			this.handleChunkBatch(message)
+			return
+		}
 		switch (message.length) {
 			//request chunk
 			case 8: {
@@ -606,6 +624,20 @@ export class Client {
 				this.y = y
 				return
 			}
+			//region level-of-detail request
+			case 16: {
+				if (message.readUInt16LE(0) !== regionLodGuard) {
+					this.destroy()
+					return
+				}
+				this.handleRegionLod(
+					message.readInt32LE(4),
+					message.readInt32LE(8),
+					message.readUInt16LE(12),
+					message.readUInt16LE(14)
+				)
+				return
+			}
 			//rank verification
 			case 1: {
 				if (message[0] > this.rank) {
@@ -620,6 +652,127 @@ export class Client {
 		}
 	}
 
+	handleChunkBatch(message) {
+		let count = message.readUInt16LE(2)
+		if (count === 0 || count !== (message.length - 6) / 8 || count > maxChunkBatchCount) {
+			this.destroy()
+			return
+		}
+		let parts = []
+		let pending = 3
+		for (let i = 0; i < count; i++) {
+			let offset = 6 + i * 8
+			let chunkX = message.readInt32LE(offset)
+			if (chunkX > maxChunkCoord || chunkX < minChunkCoord) {
+				this.destroy()
+				return
+			}
+			let chunkY = message.readInt32LE(offset + 4)
+			if (chunkY > maxChunkCoord || chunkY < minChunkCoord) {
+				this.destroy()
+				return
+			}
+			let chunkLocation = (chunkY & 0xf) << 4 | chunkX & 0xf
+			let regionId = ((chunkX >> 4) + 0x10000) + (((chunkY >> 4) + 0x10000) * 0x20000)
+			let region = this.world.getRegion(regionId)
+			if (!region.loaded) {
+				let deferredActions = this.handleUnloaded(region)
+				if (!deferredActions) return
+				let buffer = Buffer.allocUnsafe(1)
+				buffer[0] = chunkLocation
+				deferredActions.push(buffer)
+				if (++this.deferredAmount >= 100000 && this.rank < 3) {
+					this.destroy()
+					return
+				}
+				continue
+			}
+			region.lastHeld = this.server.currentTick
+			let data = region.getChunkData(chunkLocation)
+			if (parts.length && pending + 2 + data.length > maxChunkBatchBytes) {
+				this.sendChunkBatch(parts)
+				parts = []
+				pending = 3
+			}
+			parts.push(data)
+			pending += 2 + data.length
+		}
+		if (parts.length) this.sendChunkBatch(parts)
+	}
+
+	handleRegionLod(regionX, regionY, width, height) {
+		if (width === 0 || height === 0) return
+		if (width * height > maxLodRegions) {
+			this.destroy()
+			return
+		}
+		let parts = []
+		let pending = 3
+		for (let dy = 0; dy < height; dy++) {
+			for (let dx = 0; dx < width; dx++) {
+				let rx = regionX + dx
+				let ry = regionY + dy
+				if (rx > maxRegionCoord || rx < minRegionCoord || ry > maxRegionCoord || ry < minRegionCoord) continue
+				let regionId = (rx + 0x10000) + ((ry + 0x10000) * 0x20000)
+				let region = this.world.getRegion(regionId)
+				if (!region.loaded) {
+					let deferredActions = this.handleUnloaded(region)
+					if (!deferredActions) return
+					deferredActions.push(Buffer.allocUnsafe(3))
+					if (++this.deferredAmount >= 100000 && this.rank < 3) {
+						this.destroy()
+						return
+					}
+					continue
+				}
+				region.lastHeld = this.server.currentTick
+				parts.push(rx, ry, region.getLodData())
+				pending += 776
+				if (pending >= maxLodBytes) {
+					this.sendRegionLod(parts)
+					parts = []
+					pending = 3
+				}
+			}
+		}
+		if (parts.length) this.sendRegionLod(parts)
+	}
+
+	//0x0C: [u8 opcode][u16 regionCount] + regionCount * ([i32 rx][i32 ry][768 bytes])
+	sendRegionLod(parts) {
+		let count = parts.length / 3
+		let out = Buffer.allocUnsafeSlow(3 + count * 776)
+		out[0] = 0x0C
+		out.writeUInt16LE(count, 1)
+		let offset = 3
+		for (let i = 0; i < parts.length; i += 3) {
+			out.writeInt32LE(parts[i], offset)
+			out.writeInt32LE(parts[i + 1], offset + 4)
+			parts[i + 2].copy(out, offset + 8)
+			offset += 776
+		}
+		this.ws.send(out.buffer, true)
+	}
+
+	//0x0B: [u8 opcode][u16 chunkCount] + chunkCount * ([u16 byteLength][0x02 packet])
+	sendChunkBatch(parts) {
+		let total = 3
+		for (let i = 0; i < parts.length; i++) total += 2 + parts[i].length
+		//allocUnsafeSlow, same reason as Region.getChunkData
+		let out = Buffer.allocUnsafeSlow(total)
+		out[0] = 0x0B
+		out.writeUInt16LE(parts.length, 1)
+		let offset = 3
+		for (let i = 0; i < parts.length; i++) {
+			let part = parts[i]
+			out.writeUInt16LE(part.length, offset)
+			offset += 2
+			part.copy(out, offset)
+			offset += part.length
+		}
+		this.ws.send(out.buffer, true)
+	}
+
 	handleUnloaded(region) {
 		if (!region.beganLoading) {
 			if (this.rank < 3 && !this.regionloadquota.canSpend()) {
@@ -646,34 +799,64 @@ export class Client {
 		let deferredActions = this.deferredRegionActions.get(regionId)
 		this.deferredAmount -= deferredActions.length
 		this.deferredRegionActions.delete(regionId)
+		//consecutive chunk loads are batched, flushed before any other action so
+		//ordering relative to pastes/erases stays exact
+		let pendingChunks = []
+		let pendingBytes = 3
+		let flushChunks = () => {
+			if (pendingChunks.length === 0) return
+			if (pendingChunks.length === 1) {
+				this.ws.send(pendingChunks[0].buffer, true)
+			} else {
+				this.sendChunkBatch(pendingChunks)
+			}
+			pendingChunks = []
+			pendingBytes = 3
+		}
 		for (let action of deferredActions) {
 			switch (action.length) {
 				//request chunk
 				case 1: {
-					region.requestChunk(this, action[0])
+					region.lastHeld = this.server.currentTick
+					let data = region.getChunkData(action[0])
+					if (pendingChunks.length && pendingBytes + 2 + data.length > maxChunkBatchBytes) flushChunks()
+					pendingChunks.push(data)
+					pendingBytes += 2 + data.length
+					continue
+				}
+				//region level-of-detail
+				case 3: {
+					flushChunks()
+					region.lastHeld = this.server.currentTick
+					this.sendRegionLod([region.x, region.y, region.getLodData()])
 					continue
 				}
 				//set pixel
 				case 5: {
+					flushChunks()
 					region.setPixel(this, action[0], action[1], action[2], action[3], action[4])
 					continue
 				}
 				//chunk paste
 				case 769: {
+					flushChunks()
 					region.pasteChunk(action[0], action.subarray(1))
 					continue
 				}
 				//erase chunk
 				case 4: {
+					flushChunks()
 					region.eraseChunk(action[0], action[1], action[2], action[3])
 					continue
 				}
 				//protect chunk
 				case 2: {
+					flushChunks()
 					region.protectChunk(action[0], action[1])
 				}
 			}
 		}
+		flushChunks()
 	}
 
 	async handlePreWorld(message, isBinary) {

src/region/Region.js

@@ -24,6 +24,7 @@ export class Region {
     this.lastHeld = this.server.currentTick
     this.loadPromise = null
     this.dataModified = false
+    this.lodCache = null
 
     this.destroyed = false
   }
@@ -66,6 +67,38 @@ export class Region {
     this.isEmpty = false
     this.dataModified = true
     this.latestDataBuffer = null // allow gc of outdated db buffer
+    this.lodCache = null
+  }
+
+  getLodData() {
+    if (this.lodCache) return this.lodCache
+    let out = Buffer.allocUnsafeSlow(768)
+    if (this.isEmpty === true || !this.pixels) {
+      let color = this.world.bgcolor.value
+      let r = color >> 16, g = (color & 0x00ff00) >> 8, b = color & 0x0000ff
+      for (let i = 0; i < 256; i++) {
+        out[i * 3] = r
+        out[i * 3 + 1] = g
+        out[i * 3 + 2] = b
+      }
+      this.lodCache = out
+      return out
+    }
+    for (let chunkLocation = 0; chunkLocation < 256; chunkLocation++) {
+      let base = chunkLocation * 768
+      let r = 0, g = 0, b = 0
+      for (let i = 0; i < 768; i += 3) {
+        r += this.pixels[base + i]
+        g += this.pixels[base + i + 1]
+        b += this.pixels[base + i + 2]
+      }
+      let o = chunkLocation * 3
+      out[o] = (r / 256) | 0
+      out[o + 1] = (g / 256) | 0
+      out[o + 2] = (b / 256) | 0
+    }
+    this.lodCache = out
+    return out
   }
 
   updateDataBuffer() {