From c5f970d1e343362d654a79f912476af2f49c6145 Mon Sep 17 00:00:00 2001 From: Matthew Knight Date: Sun, 13 Sep 2026 14:45:42 +0200 Subject: [PATCH] Port boxzer to MicroZig as a release tool --- .github/workflows/ci.yml | 18 + .github/workflows/deploy.yml | 11 +- .gitignore | 2 +- build.zig.zon | 28 +- drivers/build.zig.zon | 10 +- examples/raspberrypi/rp2xxx/build.zig.zon | 1 - examples/texasinstruments/tm4c/build.zig.zon | 1 - modules/riscv32-common/build.zig.zon | 4 +- tools/flags/build.zig.zon | 1 - tools/release/README.md | 4 + tools/release/build.zig | 49 ++ tools/release/build.zig.zon | 17 + tools/release/scripts/test.sh | 15 + tools/release/src/Archive.zig | 433 ++++++++++++++++ tools/release/src/Manifest.zig | 381 ++++++++++++++ tools/release/src/hash_test.zig | 286 +++++++++++ tools/release/src/main.zig | 421 ++++++++++++++++ tools/release/src/tar.zig | 494 +++++++++++++++++++ tools/release/test/README.md | 20 + tools/release/test/monorepo/LICENSE | 21 + tools/release/test/monorepo/b/LICENSE | 21 + tools/release/test/monorepo/b/build.zig | 14 + tools/release/test/monorepo/b/build.zig.zon | 14 + tools/release/test/monorepo/b/src/root.zig | 10 + tools/release/test/monorepo/build.zig.zon | 13 + tools/release/test/monorepo/c/LICENSE | 21 + tools/release/test/monorepo/c/build.zig | 7 + tools/release/test/monorepo/c/build.zig.zon | 11 + tools/release/test/monorepo/c/src/file | 0 tools/release/test/monorepo/c/src/root.zig | 6 + tools/release/test/monorepo/c/src/symlink | 1 + tools/release/test/workbench/a/LICENSE | 21 + tools/release/test/workbench/a/build.zig | 15 + tools/release/test/workbench/a/build.zig.zon | 17 + tools/release/test/workbench/a/src/main.zig | 6 + tools/release/test/workbench/a/src/root.zig | 10 + tools/sorcerer/build.zig.zon | 4 +- 37 files changed, 2362 insertions(+), 46 deletions(-) create mode 100644 tools/release/README.md create mode 100644 tools/release/build.zig create mode 100644 tools/release/build.zig.zon create mode 100755 tools/release/scripts/test.sh create mode 100644 tools/release/src/Archive.zig create mode 100644 tools/release/src/Manifest.zig create mode 100644 tools/release/src/hash_test.zig create mode 100644 tools/release/src/main.zig create mode 100644 tools/release/src/tar.zig create mode 100644 tools/release/test/README.md create mode 100644 tools/release/test/monorepo/LICENSE create mode 100644 tools/release/test/monorepo/b/LICENSE create mode 100644 tools/release/test/monorepo/b/build.zig create mode 100644 tools/release/test/monorepo/b/build.zig.zon create mode 100644 tools/release/test/monorepo/b/src/root.zig create mode 100644 tools/release/test/monorepo/build.zig.zon create mode 100644 tools/release/test/monorepo/c/LICENSE create mode 100644 tools/release/test/monorepo/c/build.zig create mode 100644 tools/release/test/monorepo/c/build.zig.zon create mode 100644 tools/release/test/monorepo/c/src/file create mode 100644 tools/release/test/monorepo/c/src/root.zig create mode 120000 tools/release/test/monorepo/c/src/symlink create mode 100644 tools/release/test/workbench/a/LICENSE create mode 100644 tools/release/test/workbench/a/build.zig create mode 100644 tools/release/test/workbench/a/build.zig.zon create mode 100644 tools/release/test/workbench/a/src/main.zig create mode 100644 tools/release/test/workbench/a/src/root.zig diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35393a50a..d3f8db2dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -221,3 +221,21 @@ jobs: - name: Build Website run: zig build working-directory: website + + package-dry-run: + name: Package Dry Run + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + + - run: zig build -Doptimize=ReleaseSafe + working-directory: ./tools/release + + - name: Assemble Packages + run: ./tools/release/zig-out/bin/release -- https://github.com/ZigEmbeddedGroup/microzig/releases/download diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3eb1bbe49..ccc0b758c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,14 +22,15 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2 - with: - version: master + + - run: zig build -Doptimize=ReleaseSafe + working-directory: ./tools/release - name: Extract version - run: echo "MICROZIG_VERSION=$(zig build package -- get-version)" >> $GITHUB_ENV + run: echo "MICROZIG_VERSION=$(./tools/release/zig-out/bin/release get-version)" >> $GITHUB_ENV - name: Assemble Packages - run: zig build package -- https://github.com/ZigEmbeddedGroup/microzig/releases/download + run: ./tools/release/zig-out/bin/release -- https://github.com/ZigEmbeddedGroup/microzig/releases/download - name: Create Release Draft uses: ncipollo/release-action@v1 @@ -39,4 +40,4 @@ jobs: artifactErrorsFailBuild: true draft: true generateReleaseNotes: true - artifacts: boxzer-out/${{ env.MICROZIG_VERSION }}/* + artifacts: release-out/${{ env.MICROZIG_VERSION }}/* diff --git a/.gitignore b/.gitignore index 64d47eee7..a5bd529b5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ __pycache__/ .lldbinit .venv .zig-cache/ -boxzer-out +release-out microzig-deploy/ zig-cache/ zig-out/ diff --git a/build.zig.zon b/build.zig.zon index f64afda82..21590e21c 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -2,7 +2,7 @@ .name = .microzig, // Note: This should be changed if you fork microzig! .fingerprint = 0x605a83a849186d0f, - .version = "0.15.2", + .version = "0.17.0", .minimum_zig_version = "0.17.0-dev.1936+5a625d5f3", .dependencies = .{ .@"build-internals" = .{ .path = "build-internals" }, @@ -11,24 +11,9 @@ // tools .@"tools/esp-image" = .{ .path = "tools/esp-image" }, - .@"tools/printer" = .{ .path = "tools/printer" }, .@"tools/regz" = .{ .path = "tools/regz" }, .@"tools/uf2" = .{ .path = "tools/uf2" }, .@"tools/dfu" = .{ .path = "tools/dfu" }, - .@"tools/flags" = .{ .path = "tools/flags" }, - - // modules - .@"modules/bounded-array" = .{ .path = "modules/bounded-array" }, - .@"modules/foundation-libc" = .{ .path = "modules/foundation-libc" }, - .@"modules/freertos" = .{ .path = "modules/freertos" }, - .@"modules/lwip" = .{ .path = "modules/lwip" }, - .@"modules/network" = .{ .path = "modules/network" }, - .@"modules/riscv32-common" = .{ .path = "modules/riscv32-common" }, - .@"modules/rtt" = .{ .path = "modules/rtt" }, - .@"modules/virtual-io" = .{ .path = "modules/virtual-io" }, - - // simulators - .@"sim/aviron" = .{ .path = "sim/aviron", .lazy = true }, // ports .@"port/espressif/esp" = .{ .path = "port/espressif/esp", .lazy = true }, @@ -45,20 +30,15 @@ .@"port/texasinstruments/mspm0" = .{ .path = "port/texasinstruments/mspm0", .lazy = true }, .@"port/texasinstruments/tm4c" = .{ .path = "port/texasinstruments/tm4c", .lazy = true }, .@"port/wch/ch32v" = .{ .path = "port/wch/ch32v", .lazy = true }, + + .@"modules/riscv32-common" = .{ .path = "modules/riscv32-common" }, + .@"modules/rtt" = .{ .path = "modules/rtt" }, }, .paths = .{ "README.md", "build.zig", "build.zig.zon", "LICENSE", - "design", "tools/generate_linker_script.zig", - "tools", - "port", - "examples", - "build-internals", - "core", - "drivers", - "modules", }, } diff --git a/drivers/build.zig.zon b/drivers/build.zig.zon index 0110f3831..a72f54ed9 100644 --- a/drivers/build.zig.zon +++ b/drivers/build.zig.zon @@ -8,14 +8,6 @@ .paths = .{ "build.zig", "build.zig.zon", - "framework.zig", - "base", - "display", - "input", - "io_expander", - "led", - "sensor", - "stepper", - "wireless", + "src", }, } diff --git a/examples/raspberrypi/rp2xxx/build.zig.zon b/examples/raspberrypi/rp2xxx/build.zig.zon index af393059c..38205f7c6 100644 --- a/examples/raspberrypi/rp2xxx/build.zig.zon +++ b/examples/raspberrypi/rp2xxx/build.zig.zon @@ -12,6 +12,5 @@ "build.zig", "build.zig.zon", "src", - "scripts", }, } diff --git a/examples/texasinstruments/tm4c/build.zig.zon b/examples/texasinstruments/tm4c/build.zig.zon index 5c66d9964..c7837ea8b 100644 --- a/examples/texasinstruments/tm4c/build.zig.zon +++ b/examples/texasinstruments/tm4c/build.zig.zon @@ -6,7 +6,6 @@ .microzig = .{ .path = "../../.." }, }, .paths = .{ - "README.md", "build.zig", "build.zig.zon", "src", diff --git a/modules/riscv32-common/build.zig.zon b/modules/riscv32-common/build.zig.zon index d4f218e41..609b49c05 100644 --- a/modules/riscv32-common/build.zig.zon +++ b/modules/riscv32-common/build.zig.zon @@ -1,6 +1,6 @@ .{ - .name = .riscv32common, - .fingerprint = 0xde1ed91db6b5dcf9, + .name = .riscv32_common, + .fingerprint = 0x638e68c0bbf65ffa, .version = "0.0.0", .paths = .{ "README.md", diff --git a/tools/flags/build.zig.zon b/tools/flags/build.zig.zon index f57cc7866..49bcdbc1a 100644 --- a/tools/flags/build.zig.zon +++ b/tools/flags/build.zig.zon @@ -3,7 +3,6 @@ .version = "0.0.1", .fingerprint = 0xa32e1a54c66c4a49, .paths = .{ - "README.md", "build.zig", "build.zig.zon", "src", diff --git a/tools/release/README.md b/tools/release/README.md new file mode 100644 index 000000000..2f247d5b4 --- /dev/null +++ b/tools/release/README.md @@ -0,0 +1,4 @@ +# MicroZig release + +This tool takes MicroZig's monorepo and regenerates all the sub-packages into +their own tarballs for a release. diff --git a/tools/release/build.zig b/tools/release/build.zig new file mode 100644 index 000000000..c826ab453 --- /dev/null +++ b/tools/release/build.zig @@ -0,0 +1,49 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const release_mod = b.addModule("release", .{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }); + + const release_exe = b.addExecutable(.{ + .name = "release", + .root_module = release_mod, + }); + b.installArtifact(release_exe); + + const run_cmd = b.addRunArtifact(release_exe); + run_cmd.step.dependOn(b.getInstallStep()); + run_cmd.addPassthruArgs(); + + const run_step = b.step("run", "Run the app"); + run_step.dependOn(&run_cmd.step); + + const exe_unit_tests = b.addTest(.{ + .root_module = release_mod, + }); + + const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests); + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&run_exe_unit_tests.step); + + // Integration tests that verify hash compatibility with zig fetch + const hash_test_mod = b.addModule("hash_test", .{ + .root_source_file = b.path("src/hash_test.zig"), + .target = target, + .optimize = optimize, + }); + + const hash_tests = b.addTest(.{ + .root_module = hash_test_mod, + }); + + const run_hash_tests = b.addRunArtifact(hash_tests); + const integration_test_step = b.step("test-integration", "Run integration tests (requires zig on PATH)"); + integration_test_step.dependOn(&run_hash_tests.step); + test_step.dependOn(&run_hash_tests.step); +} diff --git a/tools/release/build.zig.zon b/tools/release/build.zig.zon new file mode 100644 index 000000000..65e59e7fb --- /dev/null +++ b/tools/release/build.zig.zon @@ -0,0 +1,17 @@ +.{ + .name = .mz_tools_release, + .version = "0.1.0", + .fingerprint = 0xa203fd280ac79711, + .paths = .{ + "build.zig", + "build.zig.zon", + "README.md", + "src", + }, + .release_ignores = .{ + "test/monorepo", + "test/monorepo/b", + "test/monorepo/c", + "test/workbench/a", + }, +} diff --git a/tools/release/scripts/test.sh b/tools/release/scripts/test.sh new file mode 100755 index 000000000..6e11c1b09 --- /dev/null +++ b/tools/release/scripts/test.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -eu + +cd test/monorepo +../../zig-out/bin/release http://localhost:8000 +python3 -m http.server 8000 --directory release-out & +sleep 1 + +cd ../workbench/a +../../../../zig/build/stage3/bin/zig fetch --save=b http://localhost:8000/0.0.0/b.tar.gz +zig build + +# clean up server +jobs -p | xargs kill diff --git a/tools/release/src/Archive.zig b/tools/release/src/Archive.zig new file mode 100644 index 000000000..372d6a188 --- /dev/null +++ b/tools/release/src/Archive.zig @@ -0,0 +1,433 @@ +files: std.array_hash_map.String(File) = .{}, + +const Archive = @This(); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; +const assert = std.debug.assert; +pub const Algo = std.crypto.hash.sha2.Sha256; +pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; + +const tar = @import("tar.zig"); +const Manifest = @import("Manifest.zig"); + +const log = std.log.scoped(.archive); + +pub const File = struct { + mode: std.Io.File.Permissions, + kind: union(enum) { + regular: []const u8, + symlink: []const u8, + }, +}; + +pub fn deinit(archive: *Archive, allocator: Allocator) void { + for (archive.files.keys(), archive.files.values()) |path, file| { + allocator.free(path); + switch (file.kind) { + .regular => |text| { + allocator.free(text); + }, + .symlink => |link_path| { + allocator.free(link_path); + }, + } + } + + archive.files.deinit(allocator); +} + +fn padding_from_size(size: usize) usize { + const mod = (512 + size) % 512; + return if (mod > 0) 512 - mod else 0; +} + +fn strip_components(path: []const u8, count: u32) ![]const u8 { + var i: usize = 0; + var c = count; + while (c > 0) : (c -= 1) { + if (std.mem.indexOfScalarPos(u8, path, i, '/')) |pos| { + i = pos + 1; + } else { + return error.TarComponentsOutsideStrippedPrefix; + } + } + return path[i..]; +} + + +fn path_to_components(allocator: Allocator, path: []const u8) ![]const []const u8 { + var list: std.ArrayList([]const u8) = .empty; + defer list.deinit(allocator); + + var it = std.mem.tokenizeScalar(u8, path, '/'); + while (it.next()) |component| + try list.append(allocator, component); + + return list.toOwnedSlice(allocator); +} + +const Dir = std.Io.Dir; + +// TODO: thread pool it +pub fn read_from_fs( + allocator: Allocator, + io: Io, + root_dir: Dir, + paths: std.array_hash_map.String(Manifest.PathOrigin), +) !Archive { + var archive = Archive{}; + errdefer archive.deinit(allocator); + + for (paths.keys(), paths.values()) |path, origin| { + switch (origin) { + .in_filesystem => { + const components = try path_to_components(allocator, path); + defer allocator.free(components); + + const basename = std.fs.path.basename(path); + const dir = if (std.fs.path.dirname(path)) |dirname| + try root_dir.openDir(io, dirname, .{}) + else + root_dir; + + const is_dir = blk: { + const stat = dir.statFile(io, basename, .{}) catch |err| { + if (err == error.IsDir) + break :blk true; + + log.err("failed to detect if is a directory: '{s}'", .{path}); + return err; + }; + break :blk stat.kind == .directory; + }; + if (is_dir) { + var collected_dir = try dir.openDir(io, basename, .{ + .iterate = true, + }); + defer collected_dir.close(io); + + var walker = try collected_dir.walk(allocator); + defer walker.deinit(); + + while (try walker.next(io)) |entry| { + switch (entry.kind) { + .directory => {}, + .file => { + var path_components: std.ArrayList([]const u8) = .empty; + defer path_components.deinit(allocator); + + try path_components.appendSlice(allocator, components); + try path_components.append(allocator, entry.path); + + const path_copy = try std.fs.path.join(allocator, path_components.items); + defer allocator.free(path_copy); + + const normalized = try normalize_path_alloc(allocator, path_copy); + + const file = try entry.dir.openFile(io, entry.basename, .{}); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var file_reader = file.reader(io, &read_buf); + const text = try file_reader.interface.allocRemaining(allocator, .limited(std.math.maxInt(usize))); + errdefer allocator.free(text); + + const file_stat = try file.stat(io); + try archive.files.put(allocator, normalized, .{ + .mode = file_stat.permissions, + .kind = .{ + .regular = text, + }, + }); + }, + .sym_link => { + var path_components: std.ArrayList([]const u8) = .empty; + defer path_components.deinit(allocator); + + try path_components.appendSlice(allocator, components); + try path_components.append(allocator, entry.path); + + const path_copy = try std.fs.path.join(allocator, path_components.items); + defer allocator.free(path_copy); + + var buf: [8000]u8 = undefined; + const link_len = try entry.dir.readLink(io, entry.basename, &buf); + const link_copy = try allocator.dupe(u8, buf[0..link_len]); + + if (std.fs.path.sep != canonical_sep) { + normalize_path(link_copy); + } + + const file = try entry.dir.openFile(io, entry.basename, .{}); + defer file.close(io); + + const normalized = try normalize_path_alloc(allocator, path_copy); + + const file_stat = try file.stat(io); + + try archive.files.put(allocator, normalized, .{ + .mode = file_stat.permissions, + .kind = .{ + .symlink = link_copy, + }, + }); + }, + else => { + // Skip file types we don't handle (e.g. block devices, + // character devices, etc.) + }, + } + } + } else { + const file = try dir.openFile(io, basename, .{}); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var file_reader = file.reader(io, &read_buf); + const text = try file_reader.interface.allocRemaining(allocator, .limited(std.math.maxInt(usize))); + errdefer allocator.free(text); + + const path_copy = try std.fs.path.join(allocator, components); + defer allocator.free(path_copy); + + const file_stat = try file.stat(io); + + const normalized = try normalize_path_alloc(allocator, path_copy); + try archive.files.put(allocator, normalized, .{ + .mode = file_stat.permissions, + .kind = .{ + .regular = text, + }, + }); + } + }, + .in_memory => |content| { + const components = try path_to_components(allocator, path); + defer allocator.free(components); + + const path_copy = try std.fs.path.join(allocator, components); + defer allocator.free(path_copy); + + const normalized = try normalize_path_alloc(allocator, path_copy); + try archive.files.put(allocator, normalized, .{ + .mode = @fromBackingInt(@intCast(0o777)), + .kind = .{ + .regular = content, + }, + }); + }, + } + } + + return archive; +} + +pub fn to_tar_gz(archive: Archive, allocator: Allocator) ![]u8 { + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + + for (archive.files.keys(), archive.files.values()) |path, file| { + const size = switch (file.kind) { + .regular => |text| text.len, + .symlink => 0, + }; + const padding = padding_from_size(size); + const header = try tar.Header.init(.{ + .path = path, + .size = size, + .typeflag = switch (file.kind) { + .regular => .regular, + .symlink => .symbolic_link, + }, + .mode = @intCast(@backingInt(file.mode)), + .linkname = switch (file.kind) { + .regular => null, + .symlink => |link| link, + }, + }); + + try buf.writer.writeAll(header.to_bytes()); + switch (file.kind) { + .regular => |text| { + try buf.writer.writeAll(text); + try buf.writer.splatBytesAll(&.{0}, @as(usize, @intCast(padding))); + }, + .symlink => {}, + } + } + + try buf.writer.splatBytesAll(&.{0}, 1024); + + var out: std.Io.Writer.Allocating = try .initCapacity(allocator, 16); + defer out.deinit(); + + var gzip_buf: [std.compress.flate.max_window_len * 2]u8 = undefined; + var gzipper: std.compress.flate.Compress = try .init(&out.writer, &gzip_buf, .gzip, .default); + + try gzipper.writer.writeAll(buf.written()); + + try gzipper.finish(); + return out.toOwnedSlice(); +} + +fn path_less_than(_: void, lhs: []const u8, rhs: []const u8) bool { + return std.mem.lessThan(u8, lhs, rhs); +} + +pub const WhatToDoWithExecutableBit = enum { + ignore_executable_bit, + include_executable_bit, +}; + +pub const multihash_function: MultihashFunction = switch (Algo) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => @compileError("unreachable"), +}; + +pub const Digest = [Algo.digest_length]u8; +pub const multihash_len = 1 + 1 + Algo.digest_length; +pub const multihash_hex_digest_len = 2 * multihash_len; +pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; +const hex_charset = "0123456789abcdef"; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub fn hex_digest(digest: Digest) MultiHashHexDigest { + var result: MultiHashHexDigest = undefined; + + result[0] = hex_charset[@backingInt(multihash_function) >> 4]; + result[1] = hex_charset[@backingInt(multihash_function) & 15]; + + result[2] = hex_charset[Algo.digest_length >> 4]; + result[3] = hex_charset[Algo.digest_length & 15]; + + for (digest, 0..) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +pub fn hash( + archive: Archive, + gpa: Allocator, + name: []const u8, + semver: std.SemanticVersion, + id: u32, +) ![]const u8 { + if (name.len > 32) + return error.NameTooLong; + + var ver_buf: [32]u8 = undefined; + const ver = try std.fmt.bufPrint(&ver_buf, "{f}", .{semver}); + + const archive_hash = try archive.hash_files(gpa); + + var hashplus: [33]u8 = undefined; + std.mem.writeInt(u32, hashplus[0..4], id, .little); + std.mem.writeInt(u32, hashplus[4..8], archive_hash.size, .little); + hashplus[8..].* = archive_hash.digest[0..25].*; + + var buf: [80]u8 = undefined; + const hashplus_str = std.base64.url_safe_no_pad.Encoder.encode(&buf, &hashplus); + + return try std.fmt.allocPrint(gpa, "{s}-{s}-{s}", .{ name, ver, hashplus_str }); +} + +const ArchiveHash = struct { + digest: Digest, + size: u32, +}; + +// TODO: threadpool this +fn hash_files( + archive: Archive, + allocator: Allocator, +) !ArchiveHash { + var paths: std.ArrayList([]const u8) = .empty; + defer paths.deinit(allocator); + + var hashes: std.ArrayList([Algo.digest_length]u8) = .empty; + defer hashes.deinit(allocator); + + try paths.appendSlice(allocator, archive.files.keys()); + try hashes.appendNTimes(allocator, undefined, paths.items.len); + std.mem.sortUnstable([]const u8, paths.items, {}, path_less_than); + + var size: u32 = 0; + for (paths.items, hashes.items) |path, *result| { + const file = archive.files.get(path).?; + var hasher = Algo.init(.{}); + hasher.update(path); + + switch (file.kind) { + .regular => |text| { + // hardcode executable bit to false + hasher.update(&.{ 0, 0 }); + hasher.update(text); + + size +%= @intCast(text.len); + }, + .symlink => |symlink| { + const link_name = try normalize_path_alloc(allocator, symlink); + hasher.update(link_name); + }, + } + hasher.final(result); + } + + var hasher = Algo.init(.{}); + for (hashes.items) |file_hash| { + hasher.update(&file_hash); + } + + const digest = hasher.finalResult(); + return ArchiveHash{ + .digest = digest, + .size = size, + }; +} + +const canonical_sep = std.fs.path.sep_posix; + +fn normalize_path_alloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { + const normalized = try arena.dupe(u8, pkg_path); + if (std.fs.path.sep == canonical_sep) return normalized; + normalize_path(normalized); + return normalized; +} + +fn normalize_path(bytes: []u8) void { + assert(std.fs.path.sep != canonical_sep); + std.mem.replaceScalar(u8, bytes, std.fs.path.sep, canonical_sep); +} + +const testing = std.testing; + +test "normalize_path_alloc" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + try testing.expectEqualStrings("./file", try normalize_path_alloc(allocator, "." ++ std.fs.path.sep_str ++ "file")); + try testing.expectEqualStrings("src/file", try normalize_path_alloc(allocator, "src/file")); + try testing.expectEqualStrings("./src/file", try normalize_path_alloc(allocator, "./src/file")); +} diff --git a/tools/release/src/Manifest.zig b/tools/release/src/Manifest.zig new file mode 100644 index 000000000..8c07c60a6 --- /dev/null +++ b/tools/release/src/Manifest.zig @@ -0,0 +1,381 @@ +allocator: Allocator, +name: []const u8, +version: std.SemanticVersion, +fingerprint: Fingerprint, +dependencies: std.array_hash_map.String(PackageInfo), +release_ignores: std.array_hash_map.String(void), +paths: std.array_hash_map.String(PathOrigin), +ast: std.zig.Ast, +/// Node indices into the AST for fields we may need to rewrite during serialize. +nodes: AstNodes, + +const Manifest = @This(); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Ast = std.zig.Ast; + +const Fingerprint = packed struct(u64) { + id: u32, + checksum: u32, +}; + +/// AST node indices needed for serialization fixups. +const AstNodes = struct { + dependencies: Ast.Node.OptionalIndex = .none, + release_ignores: Ast.Node.OptionalIndex = .none, + paths: Ast.Node.OptionalIndex = .none, + minimum_zig_version: Ast.Node.OptionalIndex = .none, + version: Ast.Node.OptionalIndex = .none, + description: Ast.Node.OptionalIndex = .none, +}; + +pub const PathOrigin = union(enum) { + in_filesystem, + in_memory: []const u8, +}; + +pub const PackageInfo = union(enum) { + local: struct { + path: []const u8, + lazy: bool, + }, + remote: struct { + url: []const u8, + hash: []const u8, + lazy: bool, + }, + + pub fn format( + info: PackageInfo, + writer: *std.Io.Writer, + ) !void { + switch (info) { + .local => |local| try writer.print("local: path={s}", .{local.path}), + .remote => |remote| try writer.print("remote: url={s} hash={s}", .{ remote.url, remote.hash }), + } + } +}; + +pub fn create_from_text(allocator: Allocator, text: []const u8) !*Manifest { + const manifest = try allocator.create(Manifest); + errdefer allocator.destroy(manifest); + + manifest.* = try from_text(allocator, text); + return manifest; +} + +pub fn from_text(allocator: Allocator, text: []const u8) !Manifest { + // The source needs to be kept alive for the lifetime of the manifest since + // Ast tokens reference into it. We dupe it so the caller's buffer can be freed. + const source = try allocator.allocSentinel(u8, text.len, 0); + @memcpy(source, text); + + var ast = try Ast.parse(allocator, source, .{ .mode = .zon }); + errdefer ast.deinit(allocator); + + if (ast.errors.len > 0) + return error.ZonParseError; + + const root_node = ast.nodeData(.root).node; + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, root_node) orelse + return error.RootIsNotObject; + + var name: []const u8 = ""; + var version_text: []const u8 = ""; + var fingerprint_val: u64 = 0; + var nodes: AstNodes = .{}; + + var paths: std.array_hash_map.String(PathOrigin) = .empty; + errdefer { + for (paths.keys()) |path| allocator.free(path); + paths.deinit(allocator); + } + + var release_ignores: std.array_hash_map.String(void) = .empty; + errdefer { + for (release_ignores.keys()) |path| allocator.free(path); + release_ignores.deinit(allocator); + } + + var dependencies: std.array_hash_map.String(PackageInfo) = .empty; + errdefer { + for (dependencies.keys(), dependencies.values()) |dep_key, info| { + allocator.free(dep_key); + switch (info) { + .local => |local| allocator.free(local.path), + .remote => |remote| { + allocator.free(remote.url); + allocator.free(remote.hash); + }, + } + } + dependencies.deinit(allocator); + } + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = ast.tokenSlice(name_token); + if (std.mem.eql(u8, field_name, "name")) { + name = try parse_enum_literal(allocator, ast, field_init); + } else if (std.mem.eql(u8, field_name, "version")) { + version_text = try parse_string_literal(allocator, ast, field_init); + nodes.version = field_init.toOptional(); + } else if (std.mem.eql(u8, field_name, "fingerprint")) { + fingerprint_val = try parse_fingerprint(ast, field_init); + } else if (std.mem.eql(u8, field_name, "dependencies")) { + nodes.dependencies = field_init.toOptional(); + try parse_dependencies(allocator, ast, field_init, &dependencies); + } else if (std.mem.eql(u8, field_name, "paths")) { + nodes.paths = field_init.toOptional(); + try parse_paths(allocator, ast, field_init, &paths); + } else if (std.mem.eql(u8, field_name, "release_ignores")) { + try parse_release_ignores(allocator, ast, field_init, &release_ignores); + } else if (std.mem.eql(u8, field_name, "minimum_zig_version")) { + nodes.minimum_zig_version = field_init.toOptional(); + } else if (std.mem.eql(u8, field_name, "description")) { + nodes.description = field_init.toOptional(); + } + } + + if (name.len == 0) return error.ProjectMissingName; + if (version_text.len == 0) return error.ProjectMissingVersion; + + const semver = try std.SemanticVersion.parse(version_text); + + return Manifest{ + .allocator = allocator, + .name = name, + .version = semver, + .fingerprint = @bitCast(fingerprint_val), + .dependencies = dependencies, + .release_ignores = release_ignores, + .paths = paths, + .ast = ast, + .nodes = nodes, + }; +} + +fn parse_enum_literal(allocator: Allocator, ast: Ast, node: Ast.Node.Index) ![]const u8 { + if (ast.nodeTag(node) != .enum_literal) + return error.ProjectNameNotEnum; + const token_bytes = ast.tokenSlice(ast.nodeMainToken(node)); + return allocator.dupe(u8, token_bytes); +} + +fn parse_string_literal(allocator: Allocator, ast: Ast, node: Ast.Node.Index) ![]const u8 { + if (ast.nodeTag(node) != .string_literal) + return error.ExpectedStringLiteral; + const token_bytes = ast.tokenSlice(ast.nodeMainToken(node)); + var aw: std.Io.Writer.Allocating = .init(allocator); + errdefer aw.deinit(); + const result = std.zig.string_literal.parseWrite(&aw.writer, token_bytes) catch return error.InvalidStringLiteral; + switch (result) { + .success => return try aw.toOwnedSlice(), + .failure => return error.InvalidStringLiteral, + } +} + +fn parse_fingerprint(ast: Ast, node: Ast.Node.Index) !u64 { + if (ast.nodeTag(node) != .number_literal) + return error.FingerprintIsNotInt; + const token_bytes = ast.tokenSlice(ast.nodeMainToken(node)); + const parsed = std.zig.parseNumberLiteral(token_bytes); + return switch (parsed) { + .int => |n| @bitCast(n), + else => error.FingerprintIsNotInt, + }; +} + +fn parse_bool(ast: Ast, node: Ast.Node.Index) !bool { + if (ast.nodeTag(node) != .identifier) + return error.ExpectedBool; + const token_bytes = ast.tokenSlice(ast.nodeMainToken(node)); + if (std.mem.eql(u8, token_bytes, "true")) return true; + if (std.mem.eql(u8, token_bytes, "false")) return false; + return error.ExpectedBool; +} + +fn parse_dependencies( + allocator: Allocator, + ast: Ast, + node: Ast.Node.Index, + dependencies: *std.array_hash_map.String(PackageInfo), +) !void { + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse return; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try allocator.dupe(u8, ast.tokenSlice(name_token)); + errdefer allocator.free(dep_name); + + var dep_buf: [2]Ast.Node.Index = undefined; + const dep_struct = ast.fullStructInit(&dep_buf, field_init) orelse continue; + + var has_path = false; + var path: []const u8 = ""; + var url: []const u8 = ""; + var hash: []const u8 = ""; + var lazy: bool = false; + + for (dep_struct.ast.fields) |dep_field| { + const dep_field_name_token = ast.firstToken(dep_field) - 2; + const dep_field_name = ast.tokenSlice(dep_field_name_token); + if (std.mem.eql(u8, dep_field_name, "path")) { + has_path = true; + path = try parse_string_literal(allocator, ast, dep_field); + } else if (std.mem.eql(u8, dep_field_name, "url")) { + url = try parse_string_literal(allocator, ast, dep_field); + } else if (std.mem.eql(u8, dep_field_name, "hash")) { + hash = try parse_string_literal(allocator, ast, dep_field); + } else if (std.mem.eql(u8, dep_field_name, "lazy")) { + lazy = try parse_bool(ast, dep_field); + } + } + + if (has_path) { + try dependencies.put(allocator, dep_name, .{ + .local = .{ .path = path, .lazy = lazy }, + }); + } else { + try dependencies.put(allocator, dep_name, .{ + .remote = .{ + .url = url, + .hash = hash, + .lazy = lazy, + }, + }); + } + } +} + +fn parse_release_ignores( + allocator: Allocator, + ast: Ast, + node: Ast.Node.Index, + ignore_paths: *std.array_hash_map.String(void), +) !void { + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse return; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parse_string_literal(allocator, ast, elem_node); + errdefer allocator.free(path_string); + try ignore_paths.put(allocator, path_string, {}); + } +} + +fn parse_paths( + allocator: Allocator, + ast: Ast, + node: Ast.Node.Index, + paths: *std.array_hash_map.String(PathOrigin), +) !void { + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse return error.ProjectPathsIsNotArray; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parse_string_literal(allocator, ast, elem_node); + errdefer allocator.free(path_string); + try paths.put(allocator, path_string, .in_filesystem); + } +} + +pub fn deinit(manifest: *Manifest) void { + manifest.allocator.free(manifest.name); + manifest.ast.deinit(manifest.allocator); + + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, info| { + manifest.allocator.free(name); + switch (info) { + .local => |local| manifest.allocator.free(local.path), + .remote => |remote| { + manifest.allocator.free(remote.url); + manifest.allocator.free(remote.hash); + }, + } + } + manifest.dependencies.deinit(manifest.allocator); + + for (manifest.paths.keys()) |path| manifest.allocator.free(path); + manifest.paths.deinit(manifest.allocator); +} + +pub const SerializeOptions = struct { + minimum_zig_version: ?[]const u8, +}; + +pub fn serialize(manifest: *Manifest, allocator: Allocator, opts: SerializeOptions) ![]const u8 { + // Use an arena for all fixup string allocations so they stay alive until + // after rendering completes. + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var fixups: Ast.Render.Fixups = .{}; + defer fixups.deinit(allocator); + + // Add or replace minimum_zig_version. + if (opts.minimum_zig_version != null) { + if (manifest.nodes.minimum_zig_version.unwrap()) |mzv_node| { + const mzv_text = try std.fmt.allocPrint(a, "\"{s}\"", .{opts.minimum_zig_version.?}); + try fixups.replace_nodes_with_string.put(allocator, mzv_node, mzv_text); + } else if (manifest.nodes.version.unwrap()) |version_node| { + const mzv_field = try std.fmt.allocPrint(a, "\n .minimum_zig_version = \"{s}\",", .{opts.minimum_zig_version.?}); + try fixups.append_string_after_node.put(allocator, version_node, mzv_field); + } + } + + // Rewrite dependencies from local to remote. + if (manifest.nodes.dependencies.unwrap()) |deps_node| { + if (manifest.dependencies.count() > 0) { + var deps_buf: std.Io.Writer.Allocating = .init(a); + try deps_buf.writer.writeAll(".{\n"); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |dep_name, info| { + switch (info) { + .remote => |remote| { + try deps_buf.writer.print( + " .{f} = .{{\n .url = \"{f}\",\n .hash = \"{f}\",\n .lazy = {},\n }},\n", + .{ + std.zig.fmtId(dep_name), + std.zig.fmtString(remote.url), + std.zig.fmtString(remote.hash), + remote.lazy, + }, + ); + }, + .local => {}, + } + } + try deps_buf.writer.writeAll(" }"); + const deps_text = try deps_buf.toOwnedSlice(); + try fixups.replace_nodes_with_string.put(allocator, deps_node, deps_text); + } + } + + // Rewrite paths. + if (manifest.nodes.paths.unwrap()) |paths_node| { + var paths_buf: std.Io.Writer.Allocating = .init(a); + try paths_buf.writer.writeAll(".{\n"); + for (manifest.paths.keys()) |path| { + try paths_buf.writer.print(" \"{f}\",\n", .{std.zig.fmtString(path)}); + } + try paths_buf.writer.writeAll(" }"); + const paths_text = try paths_buf.toOwnedSlice(); + try fixups.replace_nodes_with_string.put(allocator, paths_node, paths_text); + } + + var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + try manifest.ast.render(allocator, &aw.writer, fixups); + return try aw.toOwnedSlice(); +} + +pub fn get_file_contents(manifest: *Manifest, arena: Allocator, root_dir: std.Io.Dir, io: std.Io, path: []const u8) ![]const u8 { + const origin = manifest.paths.get(path) orelse return error.FileNotFound; + return switch (origin) { + .in_memory => |content| content, + .in_filesystem => try root_dir.readFileAlloc(io, path, arena, .limited(100 * 1024 * 1024)), + }; +} diff --git a/tools/release/src/hash_test.zig b/tools/release/src/hash_test.zig new file mode 100644 index 000000000..538955c7a --- /dev/null +++ b/tools/release/src/hash_test.zig @@ -0,0 +1,286 @@ +const std = @import("std"); +const Archive = @import("Archive.zig"); +const Manifest = @import("Manifest.zig"); + +const Io = std.Io; +const Allocator = std.mem.Allocator; + +/// A test fixture defining a package to hash. +const TestPackage = struct { + name: []const u8, + version: []const u8, + fingerprint: u64, + files: []const TestFile, +}; + +const TestFile = struct { + path: []const u8, + content: []const u8, +}; + +/// Compute the CRC32 checksum of a package name to build a valid fingerprint. +fn fingerprintFor(name: []const u8, id: u32) u64 { + const crc = std.hash.Crc32.hash(name); + // packed struct(u64) { id: u32, checksum: u32 } => checksum in high bits + return (@as(u64, crc) << 32) | @as(u64, id); +} + +/// Creates a temporary directory with the given files, returning the path. +/// Caller must clean up the directory. +fn createTempPackage( + allocator: Allocator, + io: Io, + test_dir_name: []const u8, + files: []const TestFile, +) ![]const u8 { + const tmp_path = try std.fmt.allocPrint(allocator, "/tmp/release-test-{s}", .{test_dir_name}); + // Clean up any leftover + std.Io.Dir.cwd().deleteTree(io, tmp_path) catch {}; + _ = try std.Io.Dir.cwd().createDirPathOpen(io, tmp_path, .{}); + + var pkg_dir = try std.Io.Dir.cwd().openDir(io, tmp_path, .{}); + defer pkg_dir.close(io); + + for (files) |f| { + // Create parent directories if needed + if (std.fs.path.dirname(f.path)) |dirname| { + pkg_dir.createDirPath(io, dirname) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, + }; + } + const file = try pkg_dir.createFile(io, f.path, .{}); + defer file.close(io); + var write_buf: [4096]u8 = undefined; + var writer = file.writer(io, &write_buf); + try writer.interface.writeAll(f.content); + try writer.interface.flush(); + } + + return tmp_path; +} + +/// Builds an Archive from the given package files and computes the hash, +/// then compares with `zig fetch` on the equivalent tarball. +fn verifyHashMatches( + allocator: Allocator, + io: Io, + pkg: TestPackage, +) !void { + const tmp_path = try createTempPackage(allocator, io, pkg.name, pkg.files); + defer { + std.Io.Dir.cwd().deleteTree(io, tmp_path) catch {}; + allocator.free(tmp_path); + } + + // Build the paths map for Archive.read_from_fs + var paths: std.array_hash_map.String(Manifest.PathOrigin) = .empty; + defer paths.deinit(allocator); + for (pkg.files) |f| { + try paths.put(allocator, try allocator.dupe(u8, f.path), .in_filesystem); + } + defer { + for (paths.keys()) |k| allocator.free(k); + } + + // Read the archive from the filesystem + var pkg_dir = try std.Io.Dir.cwd().openDir(io, tmp_path, .{}); + defer pkg_dir.close(io); + + var archive = try Archive.read_from_fs(allocator, io, pkg_dir, paths); + defer archive.deinit(allocator); + + // Compute the hash using release's algorithm + const name = pkg.name; + const semver = try std.SemanticVersion.parse(pkg.version); + // Fingerprint is packed struct(u64) { id: u32, checksum: u32 } + // id is the low 32 bits + const id: u32 = @truncate(pkg.fingerprint); + + const release_hash = try archive.hash(allocator, name, semver, id); + defer allocator.free(release_hash); + + // Create the tar.gz for zig fetch + const tar_gz = try archive.to_tar_gz(allocator); + defer allocator.free(tar_gz); + + // Write tar.gz to a temp file + const tar_gz_path = try std.fmt.allocPrint(allocator, "/tmp/release-test-{s}.tar.gz", .{pkg.name}); + defer { + std.Io.Dir.cwd().deleteFile(io, tar_gz_path) catch {}; + allocator.free(tar_gz_path); + } + { + const file = try std.Io.Dir.cwd().createFile(io, tar_gz_path, .{}); + defer file.close(io); + var write_buf: [4096]u8 = undefined; + var writer = file.writer(io, &write_buf); + try writer.interface.writeAll(tar_gz); + try writer.interface.flush(); + } + + // Build an environment map from the current process environment so that + // the child `zig fetch` can resolve its cache directory. + var env_map: std.process.Environ.Map = blk: { + if (@hasDecl(std.process.Environ.Block, "use_global")) { + break :blk try std.process.Environ.createMap(.{ .block = .global }, allocator); + } + // PosixBlock: build from std.c.environ + var m = std.process.Environ.Map.init(allocator); + errdefer m.deinit(); + const env_ptr: [*:null]const ?[*:0]const u8 = @ptrCast(std.c.environ); + var i: usize = 0; + while (env_ptr[i]) |entry| : (i += 1) { + const entry_str = std.mem.span(entry); + const eq_pos = std.mem.indexOfScalar(u8, entry_str, '=') orelse continue; + try m.put(entry_str[0..eq_pos], entry_str[eq_pos + 1 ..]); + } + break :blk m; + }; + defer env_map.deinit(); + + // Run `zig fetch` to get the reference hash + const result = std.process.run(allocator, io, .{ + .argv = &.{ "zig", "fetch", tar_gz_path }, + .environ_map = &env_map, + }) catch |err| { + std.debug.print("Failed to run zig fetch: {}\n", .{err}); + return err; + }; + defer { + allocator.free(result.stdout); + allocator.free(result.stderr); + } + + switch (result.term) { + .exited => |code| if (code != 0) { + std.debug.print("zig fetch failed (exit {d}):\n{s}\n", .{ code, result.stderr }); + return error.ZigFetchFailed; + }, + else => { + std.debug.print("zig fetch terminated abnormally\n", .{}); + return error.ZigFetchFailed; + }, + } + + // zig fetch outputs the hash followed by a newline + const zig_hash = std.mem.trim(u8, result.stdout, &std.ascii.whitespace); + + // When a tarball has no build.zig.zon, zig fetch uses default name="N", + // version="V", id=0xffff. Since release uses the real name/version/id from + // the manifest, we compare the underlying digest instead of the full hash + // string. The hashplus is the base64url-encoded part after "name-version-". + // It encodes: id(u32 LE) + size(u32 LE) + digest[0..25] = 33 bytes. + // base64url-no-pad of 33 bytes = 44 chars. We compare bytes [8..] which + // is the digest of file contents. + const hashplus_b64_len = 44; + if (release_hash.len < hashplus_b64_len or zig_hash.len < hashplus_b64_len) { + std.debug.print("hash string too short - release: {s}, zig: {s}\n", .{ release_hash, zig_hash }); + return error.UnexpectedHashFormat; + } + const release_b64 = release_hash[release_hash.len - hashplus_b64_len ..]; + const zig_b64 = zig_hash[zig_hash.len - hashplus_b64_len ..]; + + var release_decoded: [33]u8 = undefined; + var zig_decoded: [33]u8 = undefined; + std.base64.url_safe_no_pad.Decoder.decode(&release_decoded, release_b64) catch { + std.debug.print("failed to decode release hashplus: {s}\n", .{release_b64}); + return error.DecodeError; + }; + std.base64.url_safe_no_pad.Decoder.decode(&zig_decoded, zig_b64) catch { + std.debug.print("failed to decode zig hashplus: {s}\n", .{zig_b64}); + return error.DecodeError; + }; + + // Compare size (bytes 4..8) and digest (bytes 8..) + const release_size = std.mem.readInt(u32, release_decoded[4..8], .little); + const zig_size = std.mem.readInt(u32, zig_decoded[4..8], .little); + const release_digest = release_decoded[8..]; + const zig_digest = zig_decoded[8..]; + + if (release_size != zig_size) { + std.debug.print("Size mismatch for package '{s}':\n release: {d}\n zig: {d}\n", .{ + pkg.name, release_size, zig_size, + }); + return error.SizeMismatch; + } + + std.testing.expectEqualSlices(u8, zig_digest, release_digest) catch |err| { + std.debug.print("Digest mismatch for package '{s}':\n release: {X}\n zig: {X}\n", .{ + pkg.name, release_digest, zig_digest, + }); + return err; + }; +} + +/// Sets up a real Io.Threaded instance for tests that need process spawning. +fn runHashTest(pkg: TestPackage) !void { + const gpa = std.testing.allocator; + var threaded = Io.Threaded.init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + try verifyHashMatches(gpa, io, pkg); +} + +test "hash matches zig fetch: simple package" { + try runHashTest(.{ + .name = "simple_pkg", + .version = "0.1.0", + .fingerprint = fingerprintFor("simple_pkg", 0x12345678), + .files = &.{ + .{ .path = "build.zig", .content = "pub fn build(b: *std.Build) void {}\n" }, + .{ .path = "src/main.zig", .content = "pub fn main() void {}\n" }, + }, + }); +} + +test "hash matches zig fetch: single file" { + try runHashTest(.{ + .name = "single_file", + .version = "1.0.0", + .fingerprint = fingerprintFor("single_file", 0xdeadbeef), + .files = &.{ + .{ .path = "README.md", .content = "# Single File Package\n\nHello!\n" }, + }, + }); +} + +test "hash matches zig fetch: nested directories" { + try runHashTest(.{ + .name = "nested_pkg", + .version = "2.3.1", + .fingerprint = fingerprintFor("nested_pkg", 0xcafebabe), + .files = &.{ + .{ .path = "build.zig", .content = "const std = @import(\"std\");\npub fn build(b: *std.Build) void {}\n" }, + .{ .path = "src/root.zig", .content = "pub const root = 1;\n" }, + .{ .path = "src/deep/nested/file.zig", .content = "pub const deep = true;\n" }, + .{ .path = "LICENSE", .content = "MIT License\n\nCopyright (c) 2024\n" }, + }, + }); +} + +test "hash matches zig fetch: binary content" { + // Include some binary-like content with null bytes and high bytes + const binary_content = [_]u8{ 0x00, 0x01, 0x02, 0xFF, 0xFE, 0x00, 0x42, 0x7F }; + try runHashTest(.{ + .name = "binary_pkg", + .version = "0.0.1", + .fingerprint = fingerprintFor("binary_pkg", 0x00000001), + .files = &.{ + .{ .path = "build.zig", .content = "pub fn build(b: *std.Build) void {}\n" }, + .{ .path = "data.bin", .content = &binary_content }, + }, + }); +} + +test "hash matches zig fetch: empty files" { + try runHashTest(.{ + .name = "empty_pkg", + .version = "0.1.0", + .fingerprint = fingerprintFor("empty_pkg", 0xaaaaaaaa), + .files = &.{ + .{ .path = "build.zig", .content = "" }, + .{ .path = "empty.txt", .content = "" }, + }, + }); +} diff --git a/tools/release/src/main.zig b/tools/release/src/main.zig new file mode 100644 index 000000000..5e342f44e --- /dev/null +++ b/tools/release/src/main.zig @@ -0,0 +1,421 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const json = std.json; +const Io = std.Io; + +const Manifest = @import("Manifest.zig"); +const Archive = @import("Archive.zig"); + +const StringArrayHashMap = std.array_hash_map.String; +const StringArrayHashMapUnmanaged = std.array_hash_map.String; + +pub fn main(init: std.process.Init) !void { + const allocator = init.arena.allocator(); + const io = init.io; + const args = try init.minimal.args.toSlice(allocator); + + { + const zon_text = try std.Io.Dir.cwd().readFileAlloc(io, "build.zig.zon", allocator, .limited(0x4000)); + const zon_text_copy = try allocator.allocSentinel(u8, zon_text.len, 0); + @memcpy(zon_text_copy, zon_text); + + if (std.mem.eql(u8, args[1], "get-version")) { + const manifest = try Manifest.from_text(allocator, zon_text_copy); + var stdout_buf: [4096]u8 = undefined; + var stdout = std.Io.File.stdout().writerStreaming(io, &stdout_buf); + try stdout.interface.print("{f}\n", .{manifest.version}); + try stdout.interface.flush(); + return; + } + } + + const base_url = args[1]; + + var todo: StringArrayHashMap(void) = .empty; + + var iterable = try std.Io.Dir.cwd().openDir(io, ".", .{ .iterate = true }); + defer iterable.close(io); + + var walker = try iterable.walkSelectively(allocator); + while (try walker.next(io)) |entry| switch (entry.kind) { + .directory => if (!std.mem.startsWith(u8, entry.basename, ".") and + !std.mem.eql(u8, entry.basename, "zig-out") and + !std.mem.eql(u8, entry.basename, "zig-pkg")) + { + try walker.enter(io, entry); + }, + + .file => if (std.mem.eql(u8, entry.basename, "build.zig.zon")) { + const path = try entry.dir.realPathFileAlloc(io, "build.zig.zon", allocator); + std.log.info("found build.zig.zon: {s}", .{path}); + try todo.put(allocator, std.fs.path.dirname(path).?, {}); + }, + else => {}, + }; + + var manifests: StringArrayHashMap(*Manifest) = .empty; + var dependencies: StringArrayHashMap(StringArrayHashMapUnmanaged([]const u8)) = .empty; + var release_ignores: StringArrayHashMap(void) = .empty; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const n = try std.Io.Dir.cwd().realPathFile(io, ".", &buf); + try todo.put(allocator, try allocator.dupe(u8, buf[0..n]), {}); + } + while (todo.count() > 0) { + // pop entries from todo until it's ones we haven't visited + const root_path = while (todo.pop()) |entry| { + if (!manifests.contains(entry.key)) + break entry.key; + } else continue; + + var root_dir = try std.Io.Dir.openDirAbsolute(io, root_path, .{}); + defer root_dir.close(io); + + const zon_text = try root_dir.readFileAlloc(io, "build.zig.zon", allocator, .limited(0x4000)); + const manifest = try Manifest.create_from_text(allocator, zon_text); + + const result = try dependencies.getOrPut(allocator, root_path); + std.debug.assert(!result.found_existing); + + result.value_ptr.* = .{}; + + for (manifest.dependencies.keys(), manifest.dependencies.values()) |dep_name, dep| + switch (dep) { + .local => |local| { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const n = root_dir.realPathFile(io, local.path, &buf) catch |err| { + if (err == error.FileNotFound) + std.log.err("failed to find file: {s}", .{local.path}); + return err; + }; + const realpath = try allocator.dupe(u8, buf[0..n]); + try todo.put(allocator, realpath, {}); + try result.value_ptr.put(allocator, dep_name, realpath); + }, + .remote => {}, + }; + + for (manifest.release_ignores.keys()) |rel_path| { + std.log.info("RELEASE IGNORE: {s}", .{rel_path}); + const path = try root_dir.realPathFileAlloc(io, rel_path, allocator); + try release_ignores.put(allocator, path, {}); + } + + try manifests.put(allocator, root_path, manifest); + std.log.debug("created manifest: {s}", .{root_path}); + } + + var root_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root_n = try std.Io.Dir.cwd().realPathFile(io, ".", &root_buf); + const root_path = try allocator.dupe(u8, root_buf[0..root_n]); + + var stack = try circular_dependency_found(allocator, root_path, dependencies); + defer stack.deinit(allocator); + + if (stack.items.len > 0) { + std.log.err("Circular dependency found!", .{}); + std.log.err(" {s}", .{stack.items[0]}); + for (stack.items[1..]) |elem| + std.log.err(" -> {s}", .{elem}); + + return error.CircularDependency; + } + + const depths = try calculate_depths(allocator, manifests.keys(), dependencies); + for (depths.keys(), depths.values()) |path, depth| { + std.log.debug("{}: {s}", .{ depth, path }); + } + + var archives: StringArrayHashMap(Archive) = .empty; + var hashes: StringArrayHashMap([]const u8) = .empty; + var urls: StringArrayHashMap([]const u8) = .empty; + + const root_manifest = manifests.get(root_path).?; + // calculate urls + for (manifests.keys(), manifests.values()) |path, manifest| { + try urls.put(allocator, path, try std.fmt.allocPrint(allocator, "{s}/{f}/{s}.tar.gz", .{ + base_url, + root_manifest.version, + manifest.name, + })); + if (!manifest.paths.contains("LICENSE")) { + // Copy the license file from the root + std.log.info("{s} does not have a LICENSE file, using root LICENSE", .{path}); + try manifest.paths.put(allocator, "LICENSE", .{ + .in_memory = try root_manifest.get_file_contents(allocator, std.Io.Dir.cwd(), io, "LICENSE"), + }); + } + } + + const minimum_zig_version: []const u8 = try get_minimum_zig_version(allocator, io); + var d: isize = @intCast(std.mem.max(u32, depths.values())); + while (d > -1) : (d -= 1) { + for (depths.keys(), depths.values()) |path, depth| { + if (d == depth) { + const local_deps = dependencies.get(path).?; + var manifest = manifests.get(path).?; + for (local_deps.keys(), local_deps.values()) |dep_name, dep_path| { + const old = manifest.dependencies.get(dep_name).?; + try manifest.dependencies.put(allocator, dep_name, .{ + .remote = .{ + .url = urls.get(dep_path).?, + .hash = try std.fmt.allocPrint(allocator, "{s}", .{hashes.get(dep_path).?}), + .lazy = old.local.lazy, + }, + }); + } + + var dir = try std.Io.Dir.openDirAbsolute(io, path, .{ .iterate = true }); + defer dir.close(io); + + var archive = try Archive.read_from_fs(allocator, io, dir, manifest.paths); + if (archive.files.getPtr("build.zig.zon")) |file| { + file.kind = .{ .regular = try manifest.serialize(allocator, .{ + .minimum_zig_version = minimum_zig_version, + }) }; + } + + const hash = try archive.hash(allocator, manifest.name, manifest.version, manifest.fingerprint.id); + std.log.debug("hash for {s}: {s}", .{ manifest.name, hash }); + try hashes.put(allocator, path, hash); + try archives.put(allocator, path, archive); + } + } + } + + try std.Io.Dir.cwd().deleteTree(io, "release-out"); + var out_dir = try std.Io.Dir.cwd().createDirPathOpen(io, "release-out", .{}); + defer out_dir.close(io); + + var packages: json.ObjectMap = .empty; + for (manifests.keys(), manifests.values()) |path, manifest| { + if (release_ignores.contains(path)) { + // TODO: assert no other manifest depends on this. + std.log.info("IGNORED {s}", .{path}); + continue; + } + + const out_path = try std.fmt.allocPrint(allocator, "{f}/{s}.tar.gz", .{ + root_manifest.version, + manifest.name, + }); + + var dir = try out_dir.createDirPathOpen(io, std.fs.path.dirname(out_path).?, .{}); + defer dir.close(io); + + const file = try dir.createFile(io, std.fs.path.basename(out_path), .{}); + defer file.close(io); + + const tar_gz = try archives.get(path).?.to_tar_gz(allocator); + + var buf: [4096]u8 = undefined; + var writer = file.writer(io, &buf); + try writer.interface.writeAll(tar_gz); + try writer.interface.flush(); + + var deps: json.ObjectMap = .empty; + for (manifest.dependencies.keys(), manifest.dependencies.values()) |dep_name, info| { + switch (info) { + .remote => |remote| { + var dep: json.ObjectMap = .empty; + try dep.put(allocator, "url", .{ .string = remote.url }); + try dep.put(allocator, "hash", .{ .string = remote.hash }); + try dep.put(allocator, "lazy", .{ .bool = remote.lazy }); + + try deps.put(allocator, dep_name, .{ .object = dep }); + }, + .local => |local| { + assert(false); + _ = local; + }, + } + } + + var package: json.ObjectMap = .empty; + try package.put(allocator, "dependencies", .{ .object = deps }); + try package.put(allocator, "version", .{ .string = try std.fmt.allocPrint(allocator, "{f}", .{manifest.version}) }); + try package.put(allocator, "url", .{ .string = urls.get(path).? }); + try package.put(allocator, "hash", .{ .string = hashes.get(path).? }); + + if (manifest.nodes.description.unwrap()) |_| { + const desc_token = manifest.ast.nodeMainToken(manifest.nodes.description.unwrap().?); + const desc_bytes = manifest.ast.tokenSlice(desc_token); + var desc_aw: std.Io.Writer.Allocating = .init(allocator); + defer desc_aw.deinit(); + _ = std.zig.string_literal.parseWrite(&desc_aw.writer, desc_bytes) catch {}; + if (desc_aw.written().len > 0) + try package.put(allocator, "description", .{ .string = desc_aw.written() }); + } + + try packages.put(allocator, manifest.name, .{ .object = package }); + } + + if (manifests.get(root_path)) |manifest| { + var metadata: json.ObjectMap = .empty; + try metadata.put(allocator, "version", .{ .string = try std.fmt.allocPrint(allocator, "{f}", .{manifest.version}) }); + try metadata.put(allocator, "minimum_zig_version", .{ .string = minimum_zig_version }); + try metadata.put(allocator, "packages", .{ .object = packages }); + + const version_path = try std.fmt.allocPrint(allocator, "{f}", .{manifest.version}); + var dir = try out_dir.openDir(io, version_path, .{}); + defer dir.close(io); + + const file = try dir.createFile(io, "package-metadata.json", .{}); + defer file.close(io); + + var buf: [4096]u8 = undefined; + var writer = file.writer(io, &buf); + + const value = json.Value{ .object = metadata }; + try json.Stringify.value(value, .{ .whitespace = .indent_4 }, &writer.interface); + try writer.interface.flush(); + } +} + +fn get_minimum_zig_version(allocator: Allocator, io: Io) ![]u8 { + const result = try std.process.run(allocator, io, .{ + .argv = &.{ "zig", "env" }, + }); + defer { + allocator.free(result.stdout); + allocator.free(result.stderr); + } + + switch (result.term) { + .exited => |code| if (code != 0) return error.FailedToGetZigVersion, + else => return error.FailedToGetZigVersion, + } + + // `zig env` outputs ZON, not JSON. + const source = try allocator.allocSentinel(u8, result.stdout.len, 0); + defer allocator.free(source); + @memcpy(source, result.stdout); + + var ast = try std.zig.Ast.parse(allocator, source, .{ .mode = .zon }); + defer ast.deinit(allocator); + + const root_node = ast.nodeData(.root).node; + var buf: [2]std.zig.Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, root_node) orelse + return error.FailedToGetZigVersion; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = ast.tokenSlice(name_token); + if (std.mem.eql(u8, field_name, "version")) { + if (ast.nodeTag(field_init) != .string_literal) + return error.FailedToGetZigVersion; + const token_bytes = ast.tokenSlice(ast.nodeMainToken(field_init)); + var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + _ = std.zig.string_literal.parseWrite(&aw.writer, token_bytes) catch return error.FailedToGetZigVersion; + return allocator.dupe(u8, aw.written()); + } + } + + return error.FailedToGetZigVersion; +} + +fn calculate_depths( + allocator: Allocator, + paths: []const []const u8, + dependencies: StringArrayHashMap(StringArrayHashMapUnmanaged([]const u8)), +) !StringArrayHashMap(u32) { + var dependents: StringArrayHashMap(StringArrayHashMapUnmanaged(void)) = .empty; + defer { + //for (dependents.values()) |*d| d.deinit(); + dependents.deinit(allocator); + } + + for (paths) |path| if (dependencies.get(path)) |deps| { + for (deps.values()) |dep_path| { + const result = try dependents.getOrPut(allocator, dep_path); + if (result.found_existing == false) + result.value_ptr.* = .{}; + + try result.value_ptr.put(allocator, path, {}); + } + }; + + for (dependents.keys(), dependents.values()) |path, parents| { + std.log.debug("{s} is depended on by {}", .{ path, parents.count() }); + } + + var depths: StringArrayHashMap(u32) = .empty; + errdefer depths.deinit(allocator); + + for (paths) |path| + try calculate_depths_recursive(path, &depths, dependents, allocator); + + return depths; +} + +// walk up the dependency tree and calculate depths of each node +fn calculate_depths_recursive( + path: []const u8, + depths: *StringArrayHashMap(u32), + dependents: StringArrayHashMap(StringArrayHashMapUnmanaged(void)), + allocator: Allocator, +) !void { + // if the depth for a path is found, then we've already calculated it + if (depths.contains(path)) + return; + + const parents = dependents.get(path) orelse { + // we've found the root path + try depths.put(allocator, path, 0); + return; + }; + + var max_depth: u32 = 0; + for (parents.keys()) |parent_path| { + try calculate_depths_recursive(parent_path, depths, dependents, allocator); + max_depth = @max(max_depth, depths.get(parent_path).?); + } + + try depths.put(allocator, path, max_depth + 1); +} + +fn circular_dependency_found( + allocator: Allocator, + root_path: []const u8, + dependencies: StringArrayHashMap(StringArrayHashMapUnmanaged([]const u8)), +) !std.ArrayList([]const u8) { + var stack: std.ArrayList([]const u8) = .empty; + errdefer stack.deinit(allocator); + + _ = try circular_dependency_found_recursive(root_path, &stack, dependencies, allocator); + + return stack; +} + +fn circular_dependency_found_recursive( + path: []const u8, + stack: *std.ArrayList([]const u8), + dependencies: StringArrayHashMap(StringArrayHashMapUnmanaged([]const u8)), + allocator: Allocator, +) !bool { + for (stack.items) |elem| { + if (std.mem.eql(u8, path, elem)) { + try stack.append(allocator, path); + return true; + } + } + + try stack.append(allocator, path); + + if (dependencies.get(path)) |deps| + for (deps.values()) |dep_path| { + if (try circular_dependency_found_recursive(dep_path, stack, dependencies, allocator)) + return true; + }; + + _ = stack.pop(); + return false; +} + +test "all" { + _ = Archive; +} diff --git a/tools/release/src/tar.zig b/tools/release/src/tar.zig new file mode 100644 index 000000000..3e9cc85a3 --- /dev/null +++ b/tools/release/src/tar.zig @@ -0,0 +1,494 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const testing = std.testing; +const Allocator = std.mem.Allocator; + +// ustar tar implementation +pub const Header = extern struct { + name: [100]u8, + mode: [7:0]u8, + uid: [7:0]u8, + gid: [7:0]u8, + size: [11:0]u8, + mtime: [11:0]u8, + checksum: [7:0]u8, + typeflag: FileType, + linkname: [100]u8, + magic: [5:0]u8, + version: [2]u8, + uname: [31:0]u8, + gname: [31:0]u8, + devmajor: [7:0]u8, + devminor: [7:0]u8, + prefix: [155]u8, + pad: [12]u8, + + comptime { + std.debug.assert(@sizeOf(Header) == 512); + } + + const Self = @This(); + + const FileType = enum(u8) { + regular = '0', + hard_link = '1', + symbolic_link = '2', + character = '3', + block = '4', + directory = '5', + fifo = '6', + reserved = '7', + pax_global = 'g', + extended = 'x', + _, + }; + + const Options = struct { + typeflag: FileType, + path: []const u8, + size: u64, + mode: u32, + linkname: ?[]const u8 = null, + }; + + pub fn to_bytes(header: *const Header) *const [512]u8 { + return @ptrCast(header); + } + + pub fn init(opts: Options) !Self { + var ret = std.mem.zeroes(Self); + ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' }; + ret.version = [_:0]u8{ '0', '0' }; + ret.typeflag = opts.typeflag; + + try ret.setPath(opts.path); + try ret.setSize(opts.size); + try ret.setMtime(0); + try ret.setMode(opts.typeflag, opts.mode); + try ret.setUid(0); + try ret.setGid(0); + + if (opts.linkname) |linkname| + try ret.setLinkname(linkname); + + std.mem.copyForwards(u8, &ret.uname, "root"); + std.mem.copyForwards(u8, &ret.gname, "root"); + + try ret.updateChecksum(); + return ret; + } + + pub fn setPath(self: *Self, path: []const u8) !void { + if (path.len > 100) { + var i: usize = 100; + while (i > 0) : (i -= 1) { + if (path[i] == '/' and i < 100) + break; + } + + _ = try std.fmt.bufPrint(&self.prefix, "{s}", .{path[0..i]}); + _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]}); + } else { + _ = try std.fmt.bufPrint(&self.name, "{s}", .{path}); + } + } + + pub fn setLinkname(self: *Self, linkname: []const u8) !void { + if (linkname.len > self.linkname.len) + return error.TooBig; + + _ = try std.fmt.bufPrint(&self.linkname, "{s}", .{linkname}); + } + + pub fn setSize(self: *Self, size: u64) !void { + _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size}); + } + + pub fn get_size(header: Header) !u64 { + return std.fmt.parseUnsigned(u64, &header.size, 8); + } + + pub fn setMtime(self: *Self, mtime: u32) !void { + _ = try std.fmt.bufPrint(&self.mtime, "{o:0>11}", .{mtime}); + } + + pub fn setMode(self: *Self, filetype: FileType, perm: u32) !void { + switch (filetype) { + .regular => _ = try std.fmt.bufPrint(&self.mode, "0{o:0>6}", .{perm}), + .directory => _ = try std.fmt.bufPrint(&self.mode, "0{o:0>6}", .{perm}), + .symbolic_link => _ = try std.fmt.bufPrint(&self.mode, "0{o:0>6}", .{perm}), + else => return error.Unsupported, + } + } + + pub fn get_mode(header: Header) !u32 { + return std.fmt.parseUnsigned(u32, &header.mode, 8); + } + + fn setUid(self: *Self, uid: u32) !void { + _ = try std.fmt.bufPrint(&self.uid, "{o:0>7}", .{uid}); + } + + fn setGid(self: *Self, gid: u32) !void { + _ = try std.fmt.bufPrint(&self.gid, "{o:0>7}", .{gid}); + } + + pub fn updateChecksum(self: *Self) !void { + const offset = @offsetOf(Self, "checksum"); + var checksum: usize = 0; + for (std.mem.asBytes(self), 0..) |val, i| { + checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum))) + ' ' + else + val; + } + + _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum}); + } + + pub fn fromStat(stat: std.Io.File.Stat, path: []const u8) !Header { + if (std.mem.indexOf(u8, path, "\\") != null) return error.NeedPosixPath; + if (std.fs.path.isAbsolute(path)) return error.NeedRelPath; + + var ret = Self.init(); + ret.typeflag = switch (stat.kind) { + .file => .regular, + .directory => .directory, + else => return error.UnsupportedType, + }; + + try ret.setPath(path); + try ret.setSize(stat.size); + try ret.setMtime(0); + try ret.setMode(ret.typeflag, @intCast(@intFromEnum(stat.permissions))); + + try ret.setUid(0); + try ret.setGid(0); + + std.mem.copyForwards(u8, &ret.uname, "root"); + std.mem.copyForwards(u8, &ret.gname, "root"); + + try ret.updateChecksum(); + return ret; + } + + pub fn isBlank(self: *const Header) bool { + const block = std.mem.asBytes(self); + return for (block) |elem| { + if (elem != 0) break false; + } else true; + } +}; + +test "Header size" { + try testing.expectEqual(512, @sizeOf(Header)); +} + +pub fn instantiate( + allocator: Allocator, + dir: std.fs.Dir, + reader: anytype, + skip_depth: usize, +) !void { + var count: usize = 0; + while (true) { + const header = reader.readStruct(Header) catch |err| { + return if (err == error.EndOfStream) + if (count < 2) error.AbrubtEnd else break + else + err; + }; + + if (header.isBlank()) { + count += 1; + continue; + } else if (count > 0) { + return error.Format; + } + + var size = try std.fmt.parseUnsigned(usize, &header.size, 8); + const block_size = ((size + 511) / 512) * 512; + var components = std.ArrayList([]const u8).init(allocator); + defer components.deinit(); + + var path_it = std.mem.tokenize(u8, &header.prefix, "/\x00"); + if (header.prefix[0] != 0) { + while (path_it.next()) |component| { + try components.append(component); + } + } + + path_it = std.mem.tokenize(u8, &header.name, "/\x00"); + while (path_it.next()) |component| { + try components.append(component); + } + + const tmp_path = try std.fs.path.join(allocator, components.items); + defer allocator.free(tmp_path); + + if (skip_depth >= components.items.len) { + try reader.skipBytes(block_size, .{}); + continue; + } + + var i: usize = 0; + while (i < skip_depth) : (i += 1) { + _ = components.orderedRemove(0); + } + + const file_path = try std.fs.path.join(allocator, components.items); + defer allocator.free(file_path); + + switch (header.typeflag) { + .directory => try dir.makePath(file_path), + .pax_global => try reader.skipBytes(512, .{}), + .regular => { + const file = try dir.createFile(file_path, .{ .read = true, .truncate = true }); + defer file.close(); + const skip_size = block_size - size; + + var buf: [std.mem.page_size]u8 = undefined; + while (size > 0) { + const buffered = try reader.read(buf[0..std.math.min(size, 512)]); + try file.writeAll(buf[0..buffered]); + size -= buffered; + } + + try reader.skipBytes(skip_size, .{}); + }, + else => {}, + } + } +} + +pub fn builder(allocator: Allocator, writer: anytype) Builder(@TypeOf(writer)) { + return Builder(@TypeOf(writer)).init(allocator, writer); +} + +pub fn Builder(comptime Writer: type) type { + return struct { + writer: Writer, + arena: std.heap.ArenaAllocator, + directories: std.StringHashMap(void), + + const Self = @This(); + + pub fn init(allocator: Allocator, writer: Writer) Self { + return Self{ + .arena = std.heap.ArenaAllocator.init(allocator), + .writer = writer, + .directories = std.StringHashMap(void).init(allocator), + }; + } + + pub fn deinit(self: *Self) void { + self.directories.deinit(); + self.arena.deinit(); + } + + pub fn finish(self: *Self) !void { + try self.writer.writeByteNTimes(0, 1024); + } + + fn maybeAddDirectories( + self: *Self, + path: []const u8, + ) !void { + var i: usize = 0; + while (i < path.len) : (i += 1) { + while (path[i] != '/' and i < path.len) i += 1; + if (i >= path.len) break; + const dirpath = try self.arena.allocator().dupe(u8, path[0..i]); + if (self.directories.contains(dirpath)) continue else try self.directories.put(dirpath, {}); + + const stat = std.fs.File.Stat{ + .inode = undefined, + .size = 0, + .mode = switch (builtin.os.tag) { + .windows => 0, + else => 0o755, + }, + .kind = .Directory, + .atime = undefined, + .mtime = std.time.nanoTimestamp(), + .ctime = undefined, + }; + const allocator = self.arena.child_allocator; + const posix_dirpath = try std.mem.replaceOwned(u8, allocator, dirpath, std.fs.path.sep_str_windows, std.fs.path.sep_str_posix); + defer allocator.free(posix_dirpath); + + const header = try Header.fromStat(stat, posix_dirpath); + try self.writer.writeAll(std.mem.asBytes(&header)); + } + } + + /// prefix is a path to prepend subpath with + pub fn addFile( + self: *Self, + root: std.fs.Dir, + prefix: ?[]const u8, + subpath: []const u8, + ) !void { + const allocator = self.arena.child_allocator; + const path = if (prefix) |prefix_path| + try std.fs.path.join(allocator, &[_][]const u8{ prefix_path, subpath }) + else + subpath; + defer if (prefix != null) allocator.free(path); + + const posix_path = try std.mem.replaceOwned(u8, allocator, path, std.fs.path.sep_str_windows, std.fs.path.sep_str_posix); + defer allocator.free(posix_path); + + if (std.fs.path.dirname(posix_path)) |dirname| + try self.maybeAddDirectories(posix_path[0 .. dirname.len + 1]); + const subfile = try root.openFile(subpath, .{ .mode = .read_write }); + defer subfile.close(); + + const stat = try subfile.stat(); + const header = try Header.fromStat(stat, posix_path); + var buf: [std.mem.page_size]u8 = undefined; + + try self.writer.writeAll(std.mem.asBytes(&header)); + var counter = std.io.countingWriter(self.writer); + + while (true) { + const n = try subfile.reader().read(&buf); + if (n == 0) break; + + try counter.writer().writeAll(buf[0..n]); + } + + const padding = blk: { + const mod = counter.bytes_written % 512; + break :blk if (mod > 0) 512 - mod else 0; + }; + try self.writer.writeByteNTimes(0, @as(usize, @intCast(padding))); + } + + /// add slice of bytes as file `path` + pub fn addSlice(self: *Self, slice: []const u8, path: []const u8) !void { + const allocator = self.arena.child_allocator; + const posix_path = try std.mem.replaceOwned(u8, allocator, path, std.fs.path.sep_str_windows, std.fs.path.sep_str_posix); + defer allocator.free(posix_path); + + const stat = std.fs.File.Stat{ + .inode = undefined, + .size = slice.len, + .mode = switch (builtin.os.tag) { + .windows => 0, + else => 0o644, + }, + .kind = .File, + .atime = undefined, + .mtime = std.time.nanoTimestamp(), + .ctime = undefined, + }; + + var header = try Header.fromStat(stat, posix_path); + const padding = blk: { + const mod = slice.len % 512; + break :blk if (mod > 0) 512 - mod else 0; + }; + try self.writer.writeAll(std.mem.asBytes(&header)); + try self.writer.writeAll(slice); + try self.writer.writeByteNTimes(0, padding); + } + }; +} + +pub const PaxHeaderMap = struct { + text: []const u8, + map: std.StringHashMap([]const u8), + + const Self = @This(); + + pub fn init(allocator: Allocator, reader: anytype) !Self { + // TODO: header verification + const header = try reader.readStruct(Header); + if (header.typeflag != .pax_global) return error.NotPaxGlobalHeader; + + const size = try std.fmt.parseInt(usize, &header.size, 8); + const text = try allocator.alloc(u8, size); + errdefer allocator.free(text); + + var i: usize = 0; + while (i < size) : (i = try reader.read(text[i..])) {} + + var map = std.StringHashMap([]const u8).init(allocator); + errdefer map.deinit(); + + var it = std.mem.tokenize(u8, text, "\n"); + while (it.next()) |line| { + const begin = (std.mem.indexOf(u8, line, " ") orelse return error.BadMapEntry) + 1; + const eql = std.mem.indexOf(u8, line[begin..], "=") orelse return error.BadMapEntry; + try map.put(line[begin .. begin + eql], line[begin + eql + 1 ..]); + } + + return Self{ + .text = text, + .map = map, + }; + } + + pub fn get(self: Self, key: []const u8) ?[]const u8 { + return self.map.get(key); + } + + pub fn deinit(self: *Self) void { + self.map.allocator.free(self.text); + self.map.deinit(); + } +}; + +pub fn fileExtractor(path: []const u8, reader: anytype) FileExtractor(@TypeOf(reader)) { + return FileExtractor(@TypeOf(reader)).init(path, reader); +} + +pub fn FileExtractor(comptime ReaderType: type) type { + return struct { + path: []const u8, + internal: ReaderType, + len: ?usize, + + const Self = @This(); + + pub fn init(path: []const u8, internal: ReaderType) Self { + return Self{ + .path = path, + .internal = internal, + .len = null, + }; + } + + pub const Error = ReaderType.Error || error{ FileNotFound, EndOfStream } || std.fmt.ParseIntError; + pub const Reader = std.io.Reader(*Self, Error, read); + + pub fn read(self: *Self, buf: []u8) Error!usize { + if (self.len == null) { + while (true) { + const header = try self.internal.readStruct(Header); + for (std.mem.asBytes(&header)) |c| { + if (c != 0) break; + } else return error.FileNotFound; + const size = try std.fmt.parseInt(usize, &header.size, 8); + const name = header.name[0 .. std.mem.indexOf(u8, &header.name, "\x00") orelse header.name.len]; + if (std.mem.eql(u8, name, self.path)) { + self.len = size; + break; + } else if (size > 0) { + try self.internal.skipBytes(size + (512 - (size % 512)), .{}); + } + } + } + + const n = try self.internal.read(buf[0..std.math.min(self.len.?, buf.len)]); + self.len.? -= n; + return n; + } + + pub fn reader(self: *Self) Reader { + return .{ .context = self }; + } + }; +} diff --git a/tools/release/test/README.md b/tools/release/test/README.md new file mode 100644 index 000000000..bf15e2cd8 --- /dev/null +++ b/tools/release/test/README.md @@ -0,0 +1,20 @@ +# Package testing + +This is a bare minimum example for release, and the purpose is to validate that +what it packages is compatible with the Zig compiler. + +We have 3 packages: + +``` +A -> B -> C +``` + +`A` is the piece of software that we're working on. `B` and `C` are packages in +a monorepo that are mature. `A` depends on `B` and `B` depends on `C`. `A` uses +a url to specify where `B` is, and in the `build.zig.zon` it depends on `C` with +a path. + +Boxzer patches the `build.zig.zon` for packaging, and outputs the package +tarballs. A local server is run to serve them over http, and `A` adds `B` as a +dependency. The real test is to see that the hash for `C` from the `B` package +matches what the Zig compilers sees. diff --git a/tools/release/test/monorepo/LICENSE b/tools/release/test/monorepo/LICENSE new file mode 100644 index 000000000..9e9dc4167 --- /dev/null +++ b/tools/release/test/monorepo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Matthew Knight + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/release/test/monorepo/b/LICENSE b/tools/release/test/monorepo/b/LICENSE new file mode 100644 index 000000000..9e9dc4167 --- /dev/null +++ b/tools/release/test/monorepo/b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Matthew Knight + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/release/test/monorepo/b/build.zig b/tools/release/test/monorepo/b/build.zig new file mode 100644 index 000000000..f51b5d084 --- /dev/null +++ b/tools/release/test/monorepo/b/build.zig @@ -0,0 +1,14 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const c_dep = b.dependency("c", .{}); + _ = b.addModule("b", .{ + .root_source_file = b.path("src/root.zig"), + .imports = &.{ + .{ + .name = "c", + .module = c_dep.module("c"), + }, + }, + }); +} diff --git a/tools/release/test/monorepo/b/build.zig.zon b/tools/release/test/monorepo/b/build.zig.zon new file mode 100644 index 000000000..1ff0b3ea5 --- /dev/null +++ b/tools/release/test/monorepo/b/build.zig.zon @@ -0,0 +1,14 @@ +.{ + .name = .b, + .fingerprint = 0x71beeff9d0bb9477, + .version = "0.0.0", + .dependencies = .{ + .c = .{ .path = "../c" }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "LICENSE", + "src", + }, +} diff --git a/tools/release/test/monorepo/b/src/root.zig b/tools/release/test/monorepo/b/src/root.zig new file mode 100644 index 000000000..d3965605e --- /dev/null +++ b/tools/release/test/monorepo/b/src/root.zig @@ -0,0 +1,10 @@ +const std = @import("std"); +const c = @import("c"); + +pub fn fibonacci(n: u32) u32 { + return switch (n) { + 0 => 0, + 1 => 1, + else => c.add(fibonacci(n - 1), fibonacci(n - 2)), + }; +} diff --git a/tools/release/test/monorepo/build.zig.zon b/tools/release/test/monorepo/build.zig.zon new file mode 100644 index 000000000..52948abb2 --- /dev/null +++ b/tools/release/test/monorepo/build.zig.zon @@ -0,0 +1,13 @@ +.{ + .name = .monorepo, + .version = "0.0.0", + .fingerprint = 0x7b6bc5a951b3a10, + .dependencies = .{ + .b = .{ .path = "b" }, + .c = .{ .path = "c" }, + }, + .paths = .{ + "build.zig.zon", + "LICENSE", + }, +} diff --git a/tools/release/test/monorepo/c/LICENSE b/tools/release/test/monorepo/c/LICENSE new file mode 100644 index 000000000..9e9dc4167 --- /dev/null +++ b/tools/release/test/monorepo/c/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Matthew Knight + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/release/test/monorepo/c/build.zig b/tools/release/test/monorepo/c/build.zig new file mode 100644 index 000000000..50054334a --- /dev/null +++ b/tools/release/test/monorepo/c/build.zig @@ -0,0 +1,7 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + _ = b.addModule("c", .{ + .root_source_file = b.path("src/root.zig"), + }); +} diff --git a/tools/release/test/monorepo/c/build.zig.zon b/tools/release/test/monorepo/c/build.zig.zon new file mode 100644 index 000000000..7a644a719 --- /dev/null +++ b/tools/release/test/monorepo/c/build.zig.zon @@ -0,0 +1,11 @@ +.{ + .name = .c, + .fingerprint = 0x6b9df6fb0f093fc, + .version = "0.0.0", + .paths = .{ + "build.zig", + "build.zig.zon", + "LICENSE", + "src", + }, +} diff --git a/tools/release/test/monorepo/c/src/file b/tools/release/test/monorepo/c/src/file new file mode 100644 index 000000000..e69de29bb diff --git a/tools/release/test/monorepo/c/src/root.zig b/tools/release/test/monorepo/c/src/root.zig new file mode 100644 index 000000000..6c2e933f2 --- /dev/null +++ b/tools/release/test/monorepo/c/src/root.zig @@ -0,0 +1,6 @@ +const std = @import("std"); +const testing = std.testing; + +pub fn add(lhs: u32, rhs: u32) u32 { + return lhs + rhs; +} diff --git a/tools/release/test/monorepo/c/src/symlink b/tools/release/test/monorepo/c/src/symlink new file mode 120000 index 000000000..7b222425d --- /dev/null +++ b/tools/release/test/monorepo/c/src/symlink @@ -0,0 +1 @@ +./file \ No newline at end of file diff --git a/tools/release/test/workbench/a/LICENSE b/tools/release/test/workbench/a/LICENSE new file mode 100644 index 000000000..9e9dc4167 --- /dev/null +++ b/tools/release/test/workbench/a/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Matthew Knight + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/release/test/workbench/a/build.zig b/tools/release/test/workbench/a/build.zig new file mode 100644 index 000000000..130ff659d --- /dev/null +++ b/tools/release/test/workbench/a/build.zig @@ -0,0 +1,15 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const b_dep = b.dependency("b", .{}); + const exe = b.addExecutable(.{ + .name = "a", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + }), + }); + exe.root_module.addImport("b", b_dep.module("b")); + b.installArtifact(exe); +} diff --git a/tools/release/test/workbench/a/build.zig.zon b/tools/release/test/workbench/a/build.zig.zon new file mode 100644 index 000000000..cb6d2fe13 --- /dev/null +++ b/tools/release/test/workbench/a/build.zig.zon @@ -0,0 +1,17 @@ +.{ + .name = .a, + .fingerprint = 0xe8b7be4355d6e395, + .version = "0.0.0", + .dependencies = .{ + .b = .{ + .url = "http://localhost:8000/0.0.0/b.tar.gz", + .hash = "b-0.0.0-d5S70AkIAADUqdE_jE1-SIhn1B8JmmYOVkFD7ynKsBM4", + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "LICENSE", + "src", + }, +} diff --git a/tools/release/test/workbench/a/src/main.zig b/tools/release/test/workbench/a/src/main.zig new file mode 100644 index 000000000..457d6211e --- /dev/null +++ b/tools/release/test/workbench/a/src/main.zig @@ -0,0 +1,6 @@ +const std = @import("std"); +const b = @import("b"); + +pub fn main() !void { + std.log.info("fibonnaci of 4 is {}", .{b.fibonacci(4)}); +} diff --git a/tools/release/test/workbench/a/src/root.zig b/tools/release/test/workbench/a/src/root.zig new file mode 100644 index 000000000..ecfeade1a --- /dev/null +++ b/tools/release/test/workbench/a/src/root.zig @@ -0,0 +1,10 @@ +const std = @import("std"); +const testing = std.testing; + +export fn add(a: i32, b: i32) i32 { + return a + b; +} + +test "basic add functionality" { + try testing.expect(add(3, 7) == 10); +} diff --git a/tools/sorcerer/build.zig.zon b/tools/sorcerer/build.zig.zon index 49229fc89..5ad8a493d 100644 --- a/tools/sorcerer/build.zig.zon +++ b/tools/sorcerer/build.zig.zon @@ -1,6 +1,8 @@ .{ .name = .sorcerer, .version = "0.0.1", + .minimum_zig_version = "0.15.2", + .fingerprint = 0x5f77e0bc82473a35, .dependencies = .{ .microzig = .{ .path = "../..", @@ -26,8 +28,6 @@ .hash = "diffz-0.0.1-G2tlIQrOAQCfH15jdyaLyrMgV8eGPouFhkCeYFTmJaLk", }, }, - .minimum_zig_version = "0.15.2", - .fingerprint = 0x5f77e0bc82473a35, .paths = .{ "README.md", "build.zig",