diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index d22430e..16d99dd 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: permissions: - contents: read + contents: write jobs: build: @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-22.04, windows-latest] gui: ['OFF', 'ON'] env: PYTHONUTF8: '1' @@ -30,3 +30,26 @@ jobs: run: cmake --build cpp/build --config Release --parallel 2 - name: Test run: ctest --test-dir cpp/build -C Release --output-on-failure + - name: Report test failures + if: failure() + run: python cpp/scripts/report_test_failures.py + - name: Package Linux x86-64 + if: matrix.gui == 'ON' && runner.os == 'Linux' + run: python cpp/scripts/package.py --build cpp/build --platform linux --output packages/fear-arch00-tool-0.1.0-alpha-linux-x86_64.zip + - name: Package Windows x86-64 + if: matrix.gui == 'ON' && runner.os == 'Windows' + run: python cpp/scripts/package.py --build cpp/build --platform windows --output packages/fear-arch00-tool-0.1.0-alpha-windows-x86_64.zip + - name: Upload tested package + if: matrix.gui == 'ON' + uses: actions/upload-artifact@v4 + with: + name: package-${{ runner.os }}-x86_64 + path: packages/*.zip + if-no-files-found: error + retention-days: 7 + # Only the explicitly requested alpha release commit uploads assets. + - name: Upload alpha asset to draft release + if: matrix.gui == 'ON' && github.event_name == 'push' && github.ref == 'refs/heads/arena/2682f006-arch00tool-fear-1-arch00-ltar' && startsWith(github.event.head_commit.message, '[alpha-release]') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python cpp/scripts/upload_alpha.py diff --git a/cpp/README.md b/cpp/README.md index c08cb3b..88db93b 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -2,7 +2,7 @@ This is a first-stage LTAR v3 metadata parser with `list`, `test`, `extract`, `test-all`, `extract-all`, and `repack` commands. Raw and zlib payloads are processed in bounded chunks; extraction creates files exclusively and rejects lexical traversal paths. The parser and extractor have been tested on generated fixtures on Linux, not on real game archives or Windows. -**Not yet a replacement for the Python tool:** Complete CLI compatibility has not been established. An experimental browser GUI is available separately, but it does not implement live progress or cancellation. Pattern matching supports `*`, `?` and character classes, but has not been exhaustively compared with Python for unusual patterns. Extraction does not protect against existing directory symlinks or races: use a new private output directory. Partial files remain on error. Do not use untrusted archives. See the synthetic benchmark below; its results do not establish real-game performance. +**Not yet a replacement for the Python tool:** Complete CLI compatibility has not been established. An experimental browser GUI is available separately, with cooperative progress and cancellation. Pattern matching supports `*`, `?` and character classes, but has not been exhaustively compared with Python for unusual patterns. Extraction does not protect against existing directory symlinks or races: use a new private output directory. Partial files remain on error. Do not use untrusted archives. See the synthetic benchmark below; its results do not establish real-game performance. ## Build and test @@ -37,9 +37,9 @@ ctest --test-dir cpp/build-gui --output-on-failure -C Release On Windows, run the generated `arch00tool-gui.exe` from the configuration directory (for example `cpp\build-gui\Release\arch00tool-gui.exe` with Visual Studio). CMake fetches pinned versions of cpp-httplib and nlohmann/json from GitHub for this optional target; the CLI-only build does not fetch them. -Copy the complete URL printed by the program into a browser **on the same computer**. Enter local paths manually or use the in-app directory browser; the interface can list (first 2,000 entries), test, extract, and repack one archive, or test/extract all archives in an input directory. Single-archive extraction supports an optional case-sensitive pattern; batch extraction supports skipping existing destinations. It does not automatically open the browser, show live progress or cancel an operation. For repacking, select the output folder and append a new archive filename manually. The picker lists at most 2,000 items per directory and runs with the same local filesystem privileges as the tool. The browser waits for each operation to finish. Batch errors are reported after processing; an incomplete extraction directory remains after failures and must be inspected and removed or moved manually before retrying. Stop the server with Ctrl+C after use. Closing the tab does not stop it. +Copy the complete URL printed by the program into a browser **on the same computer**. Enter local paths manually or use the in-app directory browser; the interface can list (first 2,000 entries), test, extract, and repack one archive, or test/extract all archives in an input directory. Single-archive extraction supports an optional case-sensitive pattern; batch extraction supports skipping existing destinations. It does not automatically open the browser. For repacking, select the output folder and append a new archive filename manually. The picker lists at most 2,000 items per directory and runs with the same local filesystem privileges as the tool. The browser shows the current phase and completed-entry count, and offers cooperative cancellation while waiting. Progress resets for each phase and each archive in batch operations; it is not a byte-based percentage or time estimate. Metadata parsing, filesystem calls, syncing and publication may delay cancellation. During repack publication cancellation is rejected as too late. Extraction cancellation leaves completed and partially written files; batch extraction may retain completed archive directories and an `.incomplete` directory. Repack attempts to remove its `.partial` file when cancelled before publication, but a crash or cleanup failure can leave it behind. Batch errors are reported after processing; an incomplete extraction directory remains after failures and must be inspected and removed or moved manually before retrying. Stop the server with Ctrl+C after use. Closing the tab does not stop it. -The server binds only to IPv4 loopback, checks the host and origin, and requires a per-run token for operations. The URL contains that token: do not share it, expose the server to a network, or run it in a shared or untrusted local environment. It is not a multi-user service or a sandbox. A local process or browser extension with access to the token can invoke operations with the user's filesystem privileges. Source and destination safety limitations described above also apply. The GUI has been built and its HTTP API tested on Linux using synthetic archives; Windows, interactive browsers, and real game archives have not been tested. +The server binds only to IPv4 loopback, checks the host and origin, and requires a per-run token for operations. The URL contains that token: do not share it, expose the server to a network, or run it in a shared or untrusted local environment. It is not a multi-user service or a sandbox. A local process or browser extension with access to the token can invoke operations with the user's filesystem privileges. Source and destination safety limitations described above also apply. The GUI has been built and its HTTP API, including cooperative cancellation, tested on Linux using synthetic archives; Windows, interactive browsers, and real game archives have not been tested. ## Synthetic performance benchmark diff --git a/cpp/RELEASE.md b/cpp/RELEASE.md new file mode 100644 index 0000000..5dbc94b --- /dev/null +++ b/cpp/RELEASE.md @@ -0,0 +1,17 @@ +# fear-arch00-tool — experimental alpha + +These are pre-release builds of the C++ port. Back up your files and work on copies. The tool has not yet been validated against original F.E.A.R. archives or in-game; the GUI has not been tested interactively on all platforms. Extract files only into a new private directory without symlinks. Cancellation or errors may leave partially extracted files; inspect incomplete output before removing it. Repack verifies copied bytes but cannot guarantee game compatibility. See the repository documentation for full safety limitations. + +## Windows x86-64 + +Extract the ZIP and run `arch00tool-cpp.exe` from PowerShell or Command Prompt, or run `arch00tool-gui.exe` and open the complete URL printed in its console on the same machine. The C++ runtime distributed with Windows or Microsoft Visual C++ Redistributable may be required. + +## Linux x86-64 + +Extract the ZIP. Make the executables runnable if your extraction program does not preserve permissions (`chmod +x arch00tool-cpp arch00tool-gui`). In a terminal run `./arch00tool-cpp` or `./arch00tool-gui` and open the printed URL locally. Built on Ubuntu 22.04; requires compatible glibc (2.35 or newer), libstdc++, and zlib runtime. Other Linux distributions and CPU architectures are not guaranteed to work. + +## GUI and CLI + +The GUI binds to loopback only. Its URL contains a session token: do not share it. Closing the browser does not stop the server; press Ctrl+C in its terminal. The interface lets you browse local paths, list, test, extract, and repack. Use a new output path for repacking. Test the rebuilt archive separately. The CLI prints usage when run without arguments; the number of workers is set with `--jobs N` (1–64). + +Progress counts entries, not bytes; cancellation is cooperative and can be delayed during metadata parsing, blocking filesystem operations, syncing, or publication. Batch operations can leave completed outputs and `.incomplete` directories. Refer to the full C++ README and bilingual docs in the repository for details. diff --git a/cpp/include/arch00.hpp b/cpp/include/arch00.hpp index fb60ea3..7db4cdf 100644 --- a/cpp/include/arch00.hpp +++ b/cpp/include/arch00.hpp @@ -1,5 +1,7 @@ #pragma once #include +#include +#include #include #include #include @@ -7,6 +9,24 @@ namespace arch00 { struct Error : std::runtime_error { using std::runtime_error::runtime_error; }; +struct Cancelled : Error { Cancelled() : Error("operation cancelled") {} }; +// Optional cooperative control, owned by the caller for the operation lifetime. +struct Control { + std::atomic cancelled{false}; + std::atomic completed{0}, total{0}; + mutable std::mutex mutex; + std::string phase = "Preparing"; + bool publishing = false; + void check() const { if (cancelled.load()) throw Cancelled(); } + void start_phase(const std::string& name, std::size_t count) { + std::lock_guard lock(mutex); phase=name; completed=0; total=count; + } + bool request_cancel() { + std::lock_guard lock(mutex); + if (publishing) return false; + cancelled.store(true); return true; + } +}; struct Header { std::uint32_t version, names_size, folder_count, file_count, retail, download, crc_enabled; std::vector guid; @@ -22,14 +42,14 @@ Archive parse(const std::filesystem::path& path); namespace arch00 { // Validates stored payloads; raw_mode copies stored bytes without decompression. -void test(const std::filesystem::path& path, const Archive& archive, bool raw_mode = false, bool verbose = false, unsigned jobs = 1); +void test(const std::filesystem::path& path, const Archive& archive, bool raw_mode = false, bool verbose = false, unsigned jobs = 1, Control* control = nullptr); // Extracts all entries; files are created exclusively and are not rolled back on error. void extract(const std::filesystem::path& path, const Archive& archive, - const std::filesystem::path& output, bool raw_mode = false, bool verbose = false, unsigned jobs = 1); + const std::filesystem::path& output, bool raw_mode = false, bool verbose = false, unsigned jobs = 1, Control* control = nullptr); } namespace arch00 { struct RepackResult { std::size_t files, replaced; std::uint64_t payload_bytes; }; RepackResult repack(const std::filesystem::path& source, const std::filesystem::path& output, - const std::filesystem::path& replacements = {}, unsigned jobs = 1); + const std::filesystem::path& replacements = {}, unsigned jobs = 1, Control* control = nullptr); } diff --git a/cpp/scripts/package.py b/cpp/scripts/package.py new file mode 100644 index 0000000..43d5204 --- /dev/null +++ b/cpp/scripts/package.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Create one release ZIP from an already tested CMake GUI build.""" +import argparse +from pathlib import Path +import zipfile + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--build', required=True, type=Path) +p.add_argument('--platform', required=True, choices=('windows', 'linux')) +p.add_argument('--output', required=True, type=Path) +a = p.parse_args() +root = Path(__file__).resolve().parents[1] +exe = '.exe' if a.platform == 'windows' else '' +bindir = a.build / 'Release' if a.platform == 'windows' else a.build +files = [(bindir / ('arch00tool-cpp' + exe), 'arch00tool-cpp' + exe, 0o755), + (bindir / ('arch00tool-gui' + exe), 'arch00tool-gui' + exe, 0o755), + (root / 'RELEASE.md', 'README.md', 0o644)] +for src, _, _ in files: + if not src.is_file(): + p.error(f'missing build output: {src}') +a.output.parent.mkdir(parents=True, exist_ok=True) +with zipfile.ZipFile(a.output, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive: + for src, name, mode in files: + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = (0o100000 | mode) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, src.read_bytes()) +print(a.output) diff --git a/cpp/scripts/report_test_failures.py b/cpp/scripts/report_test_failures.py new file mode 100644 index 0000000..95ac906 --- /dev/null +++ b/cpp/scripts/report_test_failures.py @@ -0,0 +1,7 @@ +"""Expose CTest failures as a CI annotation, including when raw logs are unavailable.""" +from pathlib import Path +p = Path('cpp/build/Testing/Temporary/LastTest.log') +if p.exists(): + text = p.read_text(encoding='utf-8', errors='replace') + escaped = text[-50000:].replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + print('::error title=CTest log::' + escaped) diff --git a/cpp/scripts/upload_alpha.py b/cpp/scripts/upload_alpha.py new file mode 100644 index 0000000..900e67d --- /dev/null +++ b/cpp/scripts/upload_alpha.py @@ -0,0 +1,16 @@ +"""Upload a tested package from CI to the explicitly created alpha draft.""" +import subprocess +import sys +import time +from pathlib import Path + +release = 'v0.1.0-alpha' +package = next(Path('packages').glob('*.zip')) +for _ in range(60): + result = subprocess.run(['gh', 'release', 'view', release], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if result.returncode == 0: + break + time.sleep(5) +else: + sys.exit('Alpha draft release was not found within five minutes') +subprocess.run(['gh', 'release', 'upload', release, str(package)], check=True) diff --git a/cpp/src/gui.cpp b/cpp/src/gui.cpp index a8e5682..be8c6bc 100644 --- a/cpp/src/gui.cpp +++ b/cpp/src/gui.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,7 @@ std::string lower_ascii(std::string s) { return s; } std::string batch_run(const fs::path& directory, const fs::path& output, bool extracting, - bool raw, unsigned jobs, bool skip, bool& failed) { + bool raw, unsigned jobs, bool skip, bool& failed, arch00::Control* control) { std::vector paths; for (const auto& item : fs::directory_iterator(directory)) { if (item.is_regular_file() && lower_ascii(item.path().extension().u8string())==".arch00") paths.push_back(item.path()); @@ -32,6 +33,7 @@ std::string batch_run(const fs::path& directory, const fs::path& output, bool ex if (extracting) fs::create_directories(output); std::ostringstream result; std::size_t errors=0, skipped=0; for (const auto& path : paths) { + if (control) { control->check(); control->start_phase("Reading metadata: " + path.filename().u8string(),0); } fs::path final, partial; if (extracting) { final=output/path.filename(); final += ".extracted"; @@ -43,15 +45,19 @@ std::string batch_run(const fs::path& directory, const fs::path& output, bool ex if (fs::exists(partial)) throw arch00::Error("Incomplete destination already exists: " + partial.u8string()); } try { - auto a=arch00::parse(path); + auto a=arch00::parse(path); if (control) control->check(); if (extracting) { if (!fs::create_directory(partial)) throw arch00::Error("Could not create incomplete directory"); - arch00::extract(path,a,partial,raw,false,jobs); - fs::rename(partial,final); - } else arch00::test(path,a,raw,false,jobs); + arch00::extract(path,a,partial,raw,false,jobs,control); + if (control) { std::lock_guard guard(control->mutex); control->check(); fs::rename(partial,final); } + else fs::rename(partial,final); + } else arch00::test(path,a,raw,false,jobs,control); + if (control) control->check(); result << "OK " << path.filename().u8string() << ": " << a.entries.size() << " files\n"; - } catch (const std::exception& e) { ++errors; result << "FAIL " << path.filename().u8string() << ": " << e.what() << "\n"; } + } catch (const arch00::Cancelled&) { throw; } + catch (const std::exception& e) { ++errors; result << "FAIL " << path.filename().u8string() << ": " << e.what() << "\n"; } } + if (control) control->check(); result << "Processed " << paths.size() << " archives; skipped=" << skipped << " failed=" << errors; failed=errors!=0; return result.str(); @@ -66,7 +72,8 @@ int main() { httplib::Server server; server.set_payload_max_length(16384); server.set_default_headers({{"Cache-Control","no-store"},{"X-Content-Type-Options","nosniff"},{"Referrer-Policy","no-referrer"},{"Content-Security-Policy","default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'; base-uri 'none'; frame-ancestors 'none'; form-action 'none'"}}); - std::mutex operation; int port=0; + std::mutex operation, active_mutex; int port=0; + std::shared_ptr active; std::string active_id; const auto valid_host=[&](const httplib::Request& req) { return req.remote_addr=="127.0.0.1" && req.get_header_value("Host")=="127.0.0.1:"+std::to_string(port); }; server.Get("/", [&](const httplib::Request& req, httplib::Response& res) { if (!valid_host(req)) { res.status=403; return; } @@ -101,12 +108,36 @@ int main() { answer(200,{{"path",path.u8string()},{"parent",path.parent_path().u8string()},{"entries",entries},{"truncated",truncated}}); } catch (const std::exception& e) { answer(400,{{"message",e.what()}}); } }); + auto authorized=[&](const httplib::Request& req) { return valid_host(req) && req.get_header_value("Origin")=="http://127.0.0.1:"+std::to_string(port) && req.get_header_value("X-Arch00-Token")==token && req.get_header_value("Content-Type")=="application/json"; }; + auto control_request=[&](const httplib::Request& req, httplib::Response& res, bool cancel) { + auto respond=[&](int code, const json& value) { res.status=code; res.set_content(value.dump(-1,' ',false,json::error_handler_t::replace),"application/json"); }; + if (!authorized(req)) { respond(403,{{"message","Forbidden"}}); return; } + try { + auto id=json::parse(req.body).at("request_id").get(); + std::shared_ptr current; + { std::lock_guard guard(active_mutex); if (active && active_id==id) current=active; } + if (!current) { respond(404,{{"message","No matching active operation"}}); return; } + if (cancel) { + bool accepted=current->request_cancel(); + respond(accepted ? 200 : 409,{{"message",accepted ? "Cancellation requested" : "Publishing has begun; cancellation is too late"}}); return; + } + std::lock_guard guard(current->mutex); + respond(200,{{"phase",current->phase},{"completed",current->completed.load()},{"total",current->total.load()},{"cancelling",current->cancelled.load()},{"publishing",current->publishing}}); + } catch (const std::exception& e) { respond(400,{{"message",e.what()}}); } + }; + server.Post("/status",[&](const httplib::Request& req,httplib::Response& res){ control_request(req,res,false); }); + server.Post("/cancel",[&](const httplib::Request& req,httplib::Response& res){ control_request(req,res,true); }); server.Post("/api", [&](const httplib::Request& req, httplib::Response& res) { auto answer=[&](int status,const std::string& text) { res.status=status; res.set_content(json{{"message",text}}.dump(-1,' ',false,json::error_handler_t::replace),"application/json"); }; if (!valid_host(req) || req.get_header_value("Origin")!="http://127.0.0.1:"+std::to_string(port) || req.get_header_value("X-Arch00-Token")!=token || req.get_header_value("Content-Type")!="application/json") { answer(403,"Forbidden"); return; } std::unique_lock lock(operation,std::try_to_lock); if (!lock.owns_lock()) { answer(409,"Another operation is running"); return; } + auto control=std::make_shared(); + struct Clear { std::mutex& mutex; std::shared_ptr& active; std::string& id; ~Clear(){ std::lock_guard guard(mutex); active.reset(); id.clear(); } } clear{active_mutex,active,active_id}; try { auto data=json::parse(req.body); + auto id=data.value("request_id",std::string{}); + if (id.size()>128) throw arch00::Error("request ID is too long"); + { std::lock_guard guard(active_mutex); active=control; active_id=id; } auto action=data.at("action").get(); auto archive=data.at("archive").get(); if (archive.empty() || archive.find('\0')!=std::string::npos) throw arch00::Error("Archive path is required"); auto path=fs::u8path(archive); @@ -115,7 +146,7 @@ int main() { if (output.empty() || output.find('\0')!=std::string::npos || replace.find('\0')!=std::string::npos) throw arch00::Error("Invalid output or replacement path"); if (data.contains("jobs") && !data["jobs"].is_number_integer()) throw arch00::Error("Workers must be an integer"); auto jobs=data.value("jobs",1); if(jobs<1 || jobs>64) throw arch00::Error("Workers must be 1–64"); - auto result=arch00::repack(path,fs::u8path(output),fs::u8path(replace),static_cast(jobs)); + auto result=arch00::repack(path,fs::u8path(output),fs::u8path(replace),static_cast(jobs),control.get()); answer(200,"Repacked and verified " + std::to_string(result.files) + " files; replaced=" + std::to_string(result.replaced)); return; } if (action!="list" && action!="test" && action!="extract" && action!="test-all" && action!="extract-all") throw arch00::Error("Unknown action"); @@ -127,10 +158,11 @@ int main() { auto out=data.value("output",std::string{}); if (extracting && (out.empty() || out.find('\0')!=std::string::npos)) throw arch00::Error("Output path is required"); bool failed=false; - auto message=batch_run(path,fs::u8path(out),extracting,compression=="raw",static_cast(jobs),data.value("skip_existing",false),failed); + auto message=batch_run(path,fs::u8path(out),extracting,compression=="raw",static_cast(jobs),data.value("skip_existing",false),failed,control.get()); answer(failed ? 400 : 200,message); return; } - auto a=arch00::parse(path); + control->start_phase("Reading metadata",0); + auto a=arch00::parse(path); control->check(); if (action=="list") { std::ostringstream out; out << "LTAR v" << a.header.version << "; " << a.entries.size() << " files\n\n"; const std::size_t limit=2000; @@ -138,12 +170,13 @@ int main() { if(a.entries.size()>limit) out << "\nOnly first " << limit << " entries shown. Use the CLI for the full listing.\n"; answer(200,out.str()); return; } - if(action=="test") { arch00::test(path,a,compression=="raw",false,static_cast(jobs)); answer(200,"Validated " + std::to_string(a.entries.size()) + " files"); return; } + if(action=="test") { arch00::test(path,a,compression=="raw",false,static_cast(jobs),control.get()); control->check(); answer(200,"Validated " + std::to_string(a.entries.size()) + " files"); return; } auto output=data.at("output").get(); if(output.empty() || output.find('\0')!=std::string::npos) throw arch00::Error("Output path is required"); auto pattern=data.value("pattern",std::string{}); if (!pattern.empty()) a.entries.erase(std::remove_if(a.entries.begin(),a.entries.end(),[&](const auto& entry){return !glob_match(entry.name,pattern);}),a.entries.end()); - arch00::extract(path,a,fs::u8path(output),compression=="raw",false,static_cast(jobs)); answer(200,"Extracted " + std::to_string(a.entries.size()) + " files"); - } catch (const std::exception& e) { answer(400,e.what()); } + arch00::extract(path,a,fs::u8path(output),compression=="raw",false,static_cast(jobs),control.get()); control->check(); answer(200,"Extracted " + std::to_string(a.entries.size()) + " files"); + } catch (const arch00::Cancelled& e) { answer(409,e.what()); } + catch (const std::exception& e) { answer(400,e.what()); } }); port=server.bind_to_any_port("127.0.0.1"); if(port<0) throw arch00::Error("Could not bind loopback port"); std::cout << "Open http://127.0.0.1:" << port << "/#" << token << "\nPress Ctrl+C to stop. Do not share this URL.\n" << std::flush; diff --git a/cpp/src/gui_page.hpp b/cpp/src/gui_page.hpp index cea4698..e65ee18 100644 --- a/cpp/src/gui_page.hpp +++ b/cpp/src/gui_page.hpp @@ -4,13 +4,28 @@ static constexpr const char* gui_page = R"ARCH00HTML(

Arch00 Tool

Experimental local interface · LTAR v3

Use trusted archives and a new private output directory. No overwrite is intended, but extraction can leave partial files. Keep backups.

-

Paths refer to this computer. For batch operations, enter an input directory as the archive path. Repacking stores replacements uncompressed. A successful test does not guarantee game compatibility.

Ready
Results will appear here.
+

Paths refer to this computer. For batch operations, enter an input directory as the archive path. Repacking stores replacements uncompressed. A successful test does not guarantee game compatibility.

Ready

Results will appear here.

Choose a local path

Select an archive or navigate into a folder. For repack output, select the parent folder and append a new filename manually.

 )ARCH00HTML"; diff --git a/cpp/src/payload.cpp b/cpp/src/payload.cpp index c583117..70855a4 100644 --- a/cpp/src/payload.cpp +++ b/cpp/src/payload.cpp @@ -42,7 +42,8 @@ File create_exclusive(const std::filesystem::path& path) { void write_data(std::FILE* file, const unsigned char* data, std::size_t size) { if (file && size && std::fwrite(data, 1, size, file) != size) throw Error("failed writing output file"); } -void process(std::ifstream& in, const Entry& e, std::FILE* out, bool force_raw) { +void process(std::ifstream& in, const Entry& e, std::FILE* out, bool force_raw, Control* control) { + if (control) control->check(); if (e.stored_size == 0) { if (e.unpacked_size != 0) throw Error(e.name + ": empty stored data with nonzero unpacked size"); return; @@ -55,6 +56,7 @@ void process(std::ifstream& in, const Entry& e, std::FILE* out, bool force_raw) std::uint64_t remaining=e.stored_size, total=0; if (force_raw || e.compression == 0) { while (remaining) { + if (control) control->check(); auto n=static_cast(std::min(remaining,input.size())); if (!in.read(reinterpret_cast(input.data()),n)) throw Error("truncated payload: " + e.name); write_data(out,input.data(),n); remaining-=n; @@ -66,10 +68,12 @@ void process(std::ifstream& in, const Entry& e, std::FILE* out, bool force_raw) struct Guard { z_stream* s; ~Guard(){ inflateEnd(s); } } guard{&stream}; bool done=false; while (remaining && !done) { + if (control) control->check(); auto n=static_cast(std::min(remaining,input.size())); if (!in.read(reinterpret_cast(input.data()),n)) throw Error("truncated payload: " + e.name); remaining-=n; stream.next_in=input.data(); stream.avail_in=static_cast(n); do { + if (control) control->check(); stream.next_out=output.data(); stream.avail_out=static_cast(output.size()); int result=inflate(&stream,Z_NO_FLUSH); auto produced=output.size()-stream.avail_out; @@ -98,12 +102,13 @@ std::filesystem::path destination(const std::filesystem::path& root, std::string return result; } } -void test(const std::filesystem::path& path, const Archive& archive, bool raw_mode, bool verbose, unsigned jobs) { +void test(const std::filesystem::path& path, const Archive& archive, bool raw_mode, bool verbose, unsigned jobs, Control* control) { if (jobs < 1 || jobs > 64) throw Error("jobs must be between 1 and 64"); + if (control) { control->check(); control->start_phase("Testing",archive.entries.size()); } if (jobs == 1 || archive.entries.size() < 2) { std::ifstream in(path,std::ios::binary); if (!in) throw Error("cannot open archive"); std::size_t count=0; - for (const auto& entry:archive.entries) { process(in,entry,nullptr,raw_mode); ++count; if (verbose && entry.stored_size && count%1000==0) std::cerr << "validated " << count << "/" << archive.entries.size() << " entries\n"; } + for (const auto& entry:archive.entries) { process(in,entry,nullptr,raw_mode,control); ++count; if (control) ++control->completed; if (verbose && entry.stored_size && count%1000==0) std::cerr << "validated " << count << "/" << archive.entries.size() << " entries\n"; } return; } const auto workers=std::min(jobs,archive.entries.size()); @@ -117,7 +122,8 @@ void test(const std::filesystem::path& path, const Archive& archive, bool raw_mo while (!stop.load(std::memory_order_relaxed)) { auto index=next.fetch_add(1,std::memory_order_relaxed); if (index>=archive.entries.size()) break; - process(in,archive.entries[index],nullptr,raw_mode); + process(in,archive.entries[index],nullptr,raw_mode,control); + if (control) ++control->completed; auto count=completed.fetch_add(1,std::memory_order_relaxed)+1; if (verbose && count%1000==0) { std::lock_guard lock(mutex); @@ -137,8 +143,9 @@ void test(const std::filesystem::path& path, const Archive& archive, bool raw_mo if (failure) std::rethrow_exception(failure); } -void extract(const std::filesystem::path& path, const Archive& archive, const std::filesystem::path& output, bool raw_mode, bool verbose, unsigned jobs) { +void extract(const std::filesystem::path& path, const Archive& archive, const std::filesystem::path& output, bool raw_mode, bool verbose, unsigned jobs, Control* control) { if (jobs < 1 || jobs > 64) throw Error("jobs must be between 1 and 64"); + if (control) { control->check(); control->start_phase("Extracting",archive.entries.size()); } std::vector order; for (const auto& e:archive.entries) order.push_back(&e); std::stable_sort(order.begin(),order.end(),[](auto a, auto b){ return a->offsetoffset; }); if (jobs==1 || order.size()<2) { @@ -146,11 +153,13 @@ void extract(const std::filesystem::path& path, const Archive& archive, const st std::ifstream in(path,std::ios::binary); if (!in) throw Error("cannot open archive"); std::size_t index=0; for (auto entry:order) { + if (control) control->check(); auto target=destination(output,entry->name); std::filesystem::create_directories(target.parent_path()); auto file=create_exclusive(target); - process(in,*entry,file.get(),raw_mode); + process(in,*entry,file.get(),raw_mode,control); if (std::fflush(file.get()) != 0) throw Error("failed flushing output file"); + if (control) ++control->completed; ++index; if (verbose && (index%1000==0 || index==order.size())) std::cerr << "[" << index << "/" << order.size() << "] " << entry->name << '\n'; } return; @@ -179,8 +188,9 @@ void extract(const std::filesystem::path& path, const Archive& archive, const st auto i=next.fetch_add(1,std::memory_order_relaxed); if (i>=order.size()) break; std::filesystem::create_directories(targets[i].parent_path()); auto file=create_exclusive(targets[i]); - process(in,*order[i],file.get(),raw_mode); + process(in,*order[i],file.get(),raw_mode,control); if (std::fflush(file.get())!=0) throw Error("failed flushing output file"); + if (control) ++control->completed; auto count=completed.fetch_add(1,std::memory_order_relaxed)+1; if (verbose && (count%1000==0 || count==order.size())) { std::lock_guard lock(mutex); diff --git a/cpp/src/repack.cpp b/cpp/src/repack.cpp index 38d1b1a..74f27ee 100644 --- a/cpp/src/repack.cpp +++ b/cpp/src/repack.cpp @@ -62,21 +62,22 @@ fs::path replacement_path(const fs::path& root, std::string name) { } if(!any) throw Error("unsafe archive path: " + name); return result; } -void copy_range(std::ifstream& src, std::FILE* dst, std::uint64_t offset, std::uint64_t size) { +void copy_range(std::ifstream& src, std::FILE* dst, std::uint64_t offset, std::uint64_t size, Control* control) { seek(src,offset); std::array buffer{}; - while(size) { auto n=static_cast(std::min(size,buffer.size())); read_exact(src,buffer.data(),n); write(dst,buffer.data(),n); size-=n; } + while(size) { if (control) control->check(); auto n=static_cast(std::min(size,buffer.size())); read_exact(src,buffer.data(),n); write(dst,buffer.data(),n); size-=n; } } -void compare_range(std::ifstream& a, std::uint64_t offset_a, std::ifstream& b, std::uint64_t offset_b, std::uint64_t size) { +void compare_range(std::ifstream& a, std::uint64_t offset_a, std::ifstream& b, std::uint64_t offset_b, std::uint64_t size, Control* control) { seek(a,offset_a); seek(b,offset_b); std::array x{},y{}; - while(size) { auto n=static_cast(std::min(size,chunk)); read_exact(a,x.data(),n); read_exact(b,y.data(),n); if(!std::equal(x.begin(),x.begin()+n,y.begin())) throw Error("payload verification failed"); size-=n; } + while(size) { if (control) control->check(); auto n=static_cast(std::min(size,chunk)); read_exact(a,x.data(),n); read_exact(b,y.data(),n); if(!std::equal(x.begin(),x.begin()+n,y.begin())) throw Error("payload verification failed"); size-=n; } } struct Item { std::uint64_t old_offset, old_size, size, new_offset; fs::path replacement; }; } -RepackResult repack(const fs::path& source, const fs::path& output, const fs::path& replacements, unsigned jobs) { +RepackResult repack(const fs::path& source, const fs::path& output, const fs::path& replacements, unsigned jobs, Control* control) { + if (control) { control->check(); control->start_phase("Reading metadata",0); } if (jobs < 1 || jobs > 64) throw Error("jobs must be between 1 and 64"); if (fs::weakly_canonical(source)==fs::weakly_canonical(output)) throw Error("output path must differ from source archive"); if (fs::exists(output) || fs::is_symlink(output)) throw Error("output already exists"); - auto parsed=parse(source); auto& h=parsed.header; + auto parsed=parse(source); if (control) control->check(); auto& h=parsed.header; std::uint64_t metadata_size=48ULL+h.names_size+std::uint64_t(h.file_count)*32+std::uint64_t(h.folder_count)*16; std::ifstream src(source,std::ios::binary); if(!src) throw Error("cannot open source archive"); Bytes metadata(static_cast(metadata_size)); read_exact(src,metadata.data(),metadata.size()); @@ -104,13 +105,16 @@ RepackResult repack(const fs::path& source, const fs::path& output, const fs::pa auto temp=output; temp += ".partial"; bool owns_temp=false; try { + if (control) { control->check(); control->start_phase("Repacking",items.size()); } auto out=exclusive_file(temp); owns_temp=true; write(out.get(),metadata.data(),metadata.size()); if (jobs==1 || items.size()<2) { for(auto index:order) { + if (control) control->check(); auto& item=items[index]; - if(item.replacement.empty()) copy_range(src,out.get(),item.old_offset,item.old_size); - else { std::ifstream replacement(item.replacement,std::ios::binary); if(!replacement) throw Error("cannot open replacement"); copy_range(replacement,out.get(),0,item.size); } + if(item.replacement.empty()) copy_range(src,out.get(),item.old_offset,item.old_size,control); + else { std::ifstream replacement(item.replacement,std::ios::binary); if(!replacement) throw Error("cannot open replacement"); copy_range(replacement,out.get(),0,item.size,control); } + if (control) ++control->completed; } } else { // Every worker has its own file handles and writes a disjoint range. @@ -126,7 +130,8 @@ RepackResult repack(const fs::path& source, const fs::path& output, const fs::pa while (!stop.load(std::memory_order_relaxed)) { auto position=next_item.fetch_add(1,std::memory_order_relaxed); if (position>=order.size()) break; const auto& item=items[order[position]]; - if (!item.size) continue; + if (control) control->check(); + if (!item.size) { if (control) ++control->completed; continue; } std::ifstream replacement; std::ifstream* read_from=&input; auto offset=item.old_offset; if (!item.replacement.empty()) { @@ -139,12 +144,14 @@ RepackResult repack(const fs::path& source, const fs::path& output, const fs::pa if (!target) throw Error("repack output seek failed"); auto remaining=item.size; while (remaining) { + if (control) control->check(); auto n=static_cast(std::min(remaining,buffer.size())); read_exact(*read_from,buffer.data(),n); target.write(reinterpret_cast(buffer.data()),static_cast(n)); if (!target) throw Error("failed writing temporary archive"); remaining-=n; } + if (control) ++control->completed; } target.flush(); if (!target) throw Error("failed flushing worker stream"); target.close(); if (!target) throw Error("failed closing worker stream"); @@ -167,6 +174,7 @@ RepackResult repack(const fs::path& source, const fs::path& output, const fs::pa if(::fsync(fileno(out.get()))!=0) throw Error("failed syncing temporary archive"); #endif if(std::fclose(out.release())!=0) throw Error("failed closing temporary archive"); + if (control) { control->check(); control->start_phase("Verifying",items.size()); } auto rebuilt=parse(temp); if(rebuilt.entries.size()!=items.size() || fs::file_size(temp)!=next) throw Error("rebuilt archive size or entry count mismatch"); std::ifstream verify(temp,std::ios::binary); if(!verify) throw Error("cannot reopen temporary archive"); @@ -175,10 +183,18 @@ RepackResult repack(const fs::path& source, const fs::path& output, const fs::pa seek(src,0); Bytes source_now(original_metadata.size()); read_exact(src,source_now.data(),source_now.size()); if(source_now!=original_metadata) throw Error("source metadata changed during repack"); for(const auto& item:items) { - if(item.replacement.empty()) compare_range(src,item.old_offset,verify,item.new_offset,item.size); - else { if(fs::file_size(item.replacement)!=item.size) throw Error("replacement size changed during repack"); std::ifstream replacement(item.replacement,std::ios::binary); if(!replacement) throw Error("cannot reopen replacement"); compare_range(replacement,0,verify,item.new_offset,item.size); } + if (control) control->check(); + if(item.replacement.empty()) compare_range(src,item.old_offset,verify,item.new_offset,item.size,control); + else { if(fs::file_size(item.replacement)!=item.size) throw Error("replacement size changed during repack"); std::ifstream replacement(item.replacement,std::ios::binary); if(!replacement) throw Error("cannot reopen replacement"); compare_range(replacement,0,verify,item.new_offset,item.size,control); } + if (control) ++control->completed; } + // Windows cannot remove the temporary path while the verification handle is open. + verify.close(); if (!verify) throw Error("failed closing verification stream"); // An atomic hard-link creation fails if the destination already exists. + if (control) { + std::lock_guard lock(control->mutex); + control->check(); control->publishing=true; control->phase="Publishing"; + } fs::create_hard_link(temp,output); std::error_code cleanup_error; fs::remove(temp,cleanup_error); owns_temp=false; return {items.size(),replaced,payload_bytes}; diff --git a/cpp/tests/test_gui.py b/cpp/tests/test_gui.py index 4f6f0e5..1bdc2ad 100644 --- a/cpp/tests/test_gui.py +++ b/cpp/tests/test_gui.py @@ -1,4 +1,6 @@ +import concurrent.futures import json +import time import pathlib import re import struct @@ -78,6 +80,40 @@ def batch(name, **extra): status,result=batch('test-all',jobs=3) assert status==400 and 'failed=1' in json.loads(result)['message'] assert action('not-supported')[0]==400 + assert request('/status', b'{"request_id":"missing"}', headers)[0]==404 + assert request('/cancel', b'{"request_id":"missing"}', headers)[0]==404 + assert request('/cancel', b'{"request_id":"missing"}', {**headers,'Origin':'https://example.com'})[0]==403 + # A sparse raw payload makes a real operation long enough to request + # cancellation without using gigabytes of fixture data on disk. + large=root/'large.Arch00'; size=256*1024*1024 + big_names=b'folder\0big.bin\0second.bin\0'; data_offset=48+len(big_names)+64+16 + with large.open('wb') as f: + f.write(struct.pack('<12I',int.from_bytes(b'LTAR','little'),3,len(big_names),1,2,0,0,0,0,0,0,0)+big_names+struct.pack('