diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml new file mode 100644 index 0000000..d22430e --- /dev/null +++ b/.github/workflows/cpp.yml @@ -0,0 +1,32 @@ +name: C++ build and tests + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: ${{ matrix.os }} / GUI ${{ matrix.gui }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + gui: ['OFF', 'ON'] + env: + PYTHONUTF8: '1' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Configure + run: cmake -S cpp -B cpp/build -DARCH00_BUILD_GUI=${{ matrix.gui }} -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release + - name: Build + run: cmake --build cpp/build --config Release --parallel 2 + - name: Test + run: ctest --test-dir cpp/build -C Release --output-on-failure diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6070052..016324b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -19,12 +19,16 @@ target_include_directories(arch00_core PUBLIC include) target_compile_features(arch00_core PUBLIC cxx_std_17) add_executable(arch00tool-cpp src/main.cpp) target_link_libraries(arch00tool-cpp PRIVATE arch00_core) +if(MINGW) + target_link_options(arch00tool-cpp PRIVATE -municode) +endif() include(CTest) if(BUILD_TESTING) find_package(Python3 COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) add_test(NAME benchmark-smoke COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/benchmark.py" --cpp "$" --files 4 --size-kib 16 --repeat 1 --jobs 2) add_test(NAME parallel COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_parallel.py" "$") + add_test(NAME unicode-paths COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_unicode_paths.py" "$") add_test(NAME parser COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_parser.py" "$") add_test(NAME cli COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_cli.py" "$") add_test(NAME repack COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_repack.py" "$") diff --git a/cpp/README.md b/cpp/README.md index d7fecdf..c08cb3b 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -2,7 +2,7 @@ This is a first-stage LTAR v3 metadata parser with `list`, `test`, `extract`, `test-all`, `extract-all`, and `repack` commands. Raw and zlib payloads are processed in bounded chunks; extraction creates files exclusively and rejects lexical traversal paths. The parser and extractor have been tested on generated fixtures on Linux, not on real game archives or Windows. -**Not yet a replacement for the Python tool:** Parallel repacking and complete CLI compatibility are not implemented. An experimental browser GUI is available separately, but it does not implement live progress or cancellation. Pattern matching supports `*`, `?` and character classes, but has not been exhaustively compared with Python for unusual patterns. Extraction does not protect against existing directory symlinks or races: use a new private output directory. Partial files remain on error. Do not use untrusted archives. See the synthetic benchmark below; its results do not establish real-game performance. +**Not yet a replacement for the Python tool:** Complete CLI compatibility has not been established. An experimental browser GUI is available separately, but it does not implement live progress or cancellation. Pattern matching supports `*`, `?` and character classes, but has not been exhaustively compared with Python for unusual patterns. Extraction does not protect against existing directory symlinks or races: use a new private output directory. Partial files remain on error. Do not use untrusted archives. See the synthetic benchmark below; its results do not establish real-game performance. ## Build and test @@ -17,12 +17,12 @@ cpp/build/arch00tool-cpp test archive.Arch00 --jobs 4 cpp/build/arch00tool-cpp extract archive.Arch00 output --pattern '*.dds' --jobs 4 cpp/build/arch00tool-cpp test-all archives cpp/build/arch00tool-cpp extract-all archives extracted --skip-existing -cpp/build/arch00tool-cpp repack archive.Arch00 modded.Arch00 --replace-dir mod +cpp/build/arch00tool-cpp repack archive.Arch00 modded.Arch00 --replace-dir mod --jobs 4 ``` -`test` and `extract` accept `--compression auto|raw`. `test`, `test-all`, `extract` and `extract-all` also accept `--jobs N` (1–64, default 1) to process separate entries concurrently, with independent file handles and bounded per-worker buffers. Batch archives are still processed sequentially. Parallel extraction rejects duplicate normalized paths and file/directory target conflicts before writing; case-folding aliases may still collide at exclusive file creation. On failure, in-flight workers may finish and partial output remains. On error, parallel workers stop as soon as practical; which error is reported may differ from sequential order. Multiple readers can be slower on a hard disk or when I/O is the bottleneck. The existing Python tool remains authoritative until command behavior and real archives have been tested against the port. +`test` and `extract` accept `--compression auto|raw`. `test`, `test-all`, `extract`, `extract-all` and `repack` also accept `--jobs N` (1–64, default 1) to process separate entries concurrently, with independent file handles and bounded per-worker buffers. Batch archives are still processed sequentially. Parallel extraction rejects duplicate normalized paths and file/directory target conflicts before writing; case-folding aliases may still collide at exclusive file creation. On failure, in-flight workers may finish and partial output remains. On error, parallel workers stop as soon as practical; which error is reported may differ from sequential order. Multiple readers can be slower on a hard disk or when I/O is the bottleneck. The existing Python tool remains authoritative until command behavior and real archives have been tested against the port. -`repack` copies unchanged stored payloads and stores replacements raw, then compares all rebuilt metadata and payload bytes against their sources before publication. It does not validate zlib or game compatibility; use `test` separately. Publication uses a hard link to prevent overwriting an existing output; filesystems without hard-link support cannot publish. The output and `.partial` must not exist. Keep source and replacements unchanged while running. Repacking may remove gaps and unreferenced trailing data. +`repack` copies unchanged stored payloads and stores replacements raw, then compares all rebuilt metadata and payload bytes against their sources before publication. With `--jobs` greater than 1, independent workers copy separate entries into disjoint output ranges; metadata processing and the byte-for-byte verification pass remain sequential. Small archives, one large entry, or slow storage may not benefit. It does not validate zlib or game compatibility; use `test` separately. Publication uses a hard link to prevent overwriting an existing output; filesystems without hard-link support cannot publish. The output and `.partial` must not exist. Keep source and replacements unchanged while running. Repacking may remove gaps and unreferenced trailing data. ## Experimental browser GUI (no Qt) @@ -37,7 +37,7 @@ ctest --test-dir cpp/build-gui --output-on-failure -C Release On Windows, run the generated `arch00tool-gui.exe` from the configuration directory (for example `cpp\build-gui\Release\arch00tool-gui.exe` with Visual Studio). CMake fetches pinned versions of cpp-httplib and nlohmann/json from GitHub for this optional target; the CLI-only build does not fetch them. -Copy the complete URL printed by the program into a browser **on the same computer**. Enter local paths manually; the interface can list (first 2,000 entries), test, extract, and repack one archive, or test/extract all archives in an input directory. Single-archive extraction supports an optional case-sensitive pattern; batch extraction supports skipping existing destinations. It does not automatically open the browser, browse files, show live progress or cancel an operation. The browser waits for each operation to finish. Batch errors are reported after processing; an incomplete extraction directory remains after failures and must be inspected and removed or moved manually before retrying. Stop the server with Ctrl+C after use. Closing the tab does not stop it. +Copy the complete URL printed by the program into a browser **on the same computer**. Enter local paths manually or use the in-app directory browser; the interface can list (first 2,000 entries), test, extract, and repack one archive, or test/extract all archives in an input directory. Single-archive extraction supports an optional case-sensitive pattern; batch extraction supports skipping existing destinations. It does not automatically open the browser, show live progress or cancel an operation. For repacking, select the output folder and append a new archive filename manually. The picker lists at most 2,000 items per directory and runs with the same local filesystem privileges as the tool. The browser waits for each operation to finish. Batch errors are reported after processing; an incomplete extraction directory remains after failures and must be inspected and removed or moved manually before retrying. Stop the server with Ctrl+C after use. Closing the tab does not stop it. The server binds only to IPv4 loopback, checks the host and origin, and requires a per-run token for operations. The URL contains that token: do not share it, expose the server to a network, or run it in a shared or untrusted local environment. It is not a multi-user service or a sandbox. A local process or browser extension with access to the token can invoke operations with the user's filesystem privileges. Source and destination safety limitations described above also apply. The GUI has been built and its HTTP API tested on Linux using synthetic archives; Windows, interactive browsers, and real game archives have not been tested. @@ -49,6 +49,10 @@ After building the C++ CLI, run: python cpp/benchmarks/benchmark.py --cpp cpp/build/arch00tool-cpp --files 48 --size-kib 1024 --repeat 3 --jobs 4 ``` -The benchmark creates a temporary LTAR v3 archive with deterministic mixed raw/zlib entries, runs Python and C++ `test`, `extract` and single-worker `repack`, and reports median wall times and nominal unpacked MiB/s. It SHA-256-checks extracted files and independently extracts every repacked archive with the Python tool, outside the timed section. Temporary files are deleted when the run ends. `--workdir` selects the parent directory for testing a particular drive; the temporary directory requires enough free disk space for multiple copies of the generated data. The benchmark limits the generated unpacked size to 1 GiB. +The benchmark creates a temporary LTAR v3 archive with deterministic mixed raw/zlib entries, runs Python and C++ `test`, `extract` and `repack`, and reports median wall times and nominal unpacked MiB/s. It SHA-256-checks extracted files and independently extracts every repacked archive with the Python tool, outside the timed section. Temporary files are deleted when the run ends. `--workdir` selects the parent directory for testing a particular drive; the temporary directory requires enough free disk space for multiple copies of the generated data. The benchmark limits the generated unpacked size to 1 GiB. Measurements include process startup and are influenced by OS caching, CPU, filesystem, compression ratio, number/size of files and background activity. The runs occur in a fixed order and are not cold-cache or statistically controlled. Nominal unpacked MiB/s is not disk throughput, especially for repack which copies stored bytes. Use multiple runs and representative real archives before drawing performance conclusions. The tool never modifies real archives. Benchmark generation requires Python 3.9 or newer. + +## Continuous integration and Unicode paths + +`.github/workflows/cpp.yml` configures, builds, and runs the C++ tests with and without the optional GUI on Linux and Windows GitHub runners. The Windows CLI receives command-line arguments as UTF-16 and converts them to UTF-8 for the archive core; names printed by the CLI are UTF-8. Automated Unicode-path fixtures cover input, output, patterns, replacements, and batch operations. Windows console font and terminal encoding settings may still affect display. CI does not establish compatibility with actual game archives or test the GUI in an interactive browser. diff --git a/cpp/benchmarks/benchmark.py b/cpp/benchmarks/benchmark.py index bb301df..f061544 100644 --- a/cpp/benchmarks/benchmark.py +++ b/cpp/benchmarks/benchmark.py @@ -105,9 +105,6 @@ def main(): print('Median wall time; throughput is unpacked MiB/s, not disk throughput.') for label, command, jobs in implementations: for mode in ('test', 'extract', 'repack'): - # Repack has no jobs option; avoid reporting the same single-threaded operation twice. - if mode == 'repack' and jobs and jobs[-1] != '1': - continue samples = [] for iteration in range(args.repeat): target = root / f'result-{iteration}' @@ -116,7 +113,7 @@ def main(): elif mode == 'extract': cmd = command + ['extract', str(archive), str(target)] + jobs else: - cmd = command + ['repack', str(archive), str(target)] + cmd = command + ['repack', str(archive), str(target)] + jobs samples.append(execute(cmd)) if mode == 'extract': verify_tree(target, expected) diff --git a/cpp/include/arch00.hpp b/cpp/include/arch00.hpp index 7d106da..fb60ea3 100644 --- a/cpp/include/arch00.hpp +++ b/cpp/include/arch00.hpp @@ -31,5 +31,5 @@ void extract(const std::filesystem::path& path, const Archive& archive, namespace arch00 { struct RepackResult { std::size_t files, replaced; std::uint64_t payload_bytes; }; RepackResult repack(const std::filesystem::path& source, const std::filesystem::path& output, - const std::filesystem::path& replacements = {}); + const std::filesystem::path& replacements = {}, unsigned jobs = 1); } diff --git a/cpp/src/gui.cpp b/cpp/src/gui.cpp index 04b4bc6..a8e5682 100644 --- a/cpp/src/gui.cpp +++ b/cpp/src/gui.cpp @@ -25,9 +25,9 @@ std::string batch_run(const fs::path& directory, const fs::path& output, bool ex bool raw, unsigned jobs, bool skip, bool& failed) { std::vector paths; for (const auto& item : fs::directory_iterator(directory)) { - if (item.is_regular_file() && lower_ascii(item.path().extension().string())==".arch00") paths.push_back(item.path()); + if (item.is_regular_file() && lower_ascii(item.path().extension().u8string())==".arch00") paths.push_back(item.path()); } - std::sort(paths.begin(),paths.end(),[](const auto& a,const auto& b) { return lower_ascii(a.filename().string()) < lower_ascii(b.filename().string()); }); + std::sort(paths.begin(),paths.end(),[](const auto& a,const auto& b) { return lower_ascii(a.filename().u8string()) < lower_ascii(b.filename().u8string()); }); if (paths.empty()) throw arch00::Error("No .Arch00 archives found in input directory"); if (extracting) fs::create_directories(output); std::ostringstream result; std::size_t errors=0, skipped=0; @@ -72,6 +72,35 @@ int main() { if (!valid_host(req)) { res.status=403; return; } res.set_content(gui_page,"text/html; charset=utf-8"); }); + server.Post("/browse", [&](const httplib::Request& req, httplib::Response& res) { + auto answer=[&](int status,const json& result) { res.status=status; res.set_content(result.dump(-1,' ',false,json::error_handler_t::replace),"application/json"); }; + if (!valid_host(req) || req.get_header_value("Origin")!="http://127.0.0.1:"+std::to_string(port) || req.get_header_value("X-Arch00-Token")!=token || req.get_header_value("Content-Type")!="application/json") { answer(403,{{"message","Forbidden"}}); return; } + try { + auto data=json::parse(req.body); auto text=data.value("path",std::string{}); + if (text.find('\0')!=std::string::npos) throw arch00::Error("Invalid directory path"); + fs::path path=text.empty() ? fs::current_path() : fs::u8path(text); + if (fs::is_regular_file(path)) path=path.parent_path(); + path=fs::absolute(path).lexically_normal(); + if (!fs::is_directory(path)) throw arch00::Error("Directory does not exist"); + std::vector> items; bool truncated=false; + constexpr std::size_t max_items=2000; + for (const auto& entry : fs::directory_iterator(path)) { + std::error_code ec; + const bool directory=entry.is_directory(ec); if (ec) continue; + if (!directory && !entry.is_regular_file(ec)) continue; + if (ec) continue; + if (items.size()==max_items) { truncated=true; break; } + items.emplace_back(entry.path(),directory); + } + std::sort(items.begin(),items.end(),[](const auto& a,const auto& b) { + if (a.second!=b.second) return a.second>b.second; + return lower_ascii(a.first.filename().u8string())(), replace=data.value("replace",std::string{}); if (output.empty() || output.find('\0')!=std::string::npos || replace.find('\0')!=std::string::npos) throw arch00::Error("Invalid output or replacement path"); - auto result=arch00::repack(path,fs::u8path(output),fs::u8path(replace)); + if (data.contains("jobs") && !data["jobs"].is_number_integer()) throw arch00::Error("Workers must be an integer"); + auto jobs=data.value("jobs",1); if(jobs<1 || jobs>64) throw arch00::Error("Workers must be 1–64"); + auto result=arch00::repack(path,fs::u8path(output),fs::u8path(replace),static_cast(jobs)); answer(200,"Repacked and verified " + std::to_string(result.files) + " files; replaced=" + std::to_string(result.replaced)); return; } if (action!="list" && action!="test" && action!="extract" && action!="test-all" && action!="extract-all") throw arch00::Error("Unknown action"); diff --git a/cpp/src/gui_page.hpp b/cpp/src/gui_page.hpp index 6d81114..cea4698 100644 --- a/cpp/src/gui_page.hpp +++ b/cpp/src/gui_page.hpp @@ -2,14 +2,38 @@ static constexpr const char* gui_page = R"ARCH00HTML( Arch00 Tool

Arch00 Tool

Experimental local interface · LTAR v3

Use trusted archives and a new private output directory. No overwrite is intended, but extraction can leave partial files. Keep backups.

-

Paths refer to this computer. For batch operations, enter an input directory as the archive path. Repacking stores replacements uncompressed. A successful test does not guarantee game compatibility.

Ready
Results will appear here.
+

Paths refer to this computer. For batch operations, enter an input directory as the archive path. Repacking stores replacements uncompressed. A successful test does not guarantee game compatibility.

Ready
Results will appear here.
+

Choose a local path

Select an archive or navigate into a folder. For repack output, select the parent folder and append a new filename manually.