From e9d5bae4cccf7156859cd6c6f9e6065063a465e7 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 17 Oct 2026 14:02:59 +0800 Subject: [PATCH 1/5] Fix graph hydration delivery and page reference rendering --- app/application.ml | 263 +++++++----------- app/journal_graph_runtime.ml | 123 +++++--- app/journal_graph_transport.ml | 21 +- ...6-10-17-unify-worker-request-scheduling.md | 39 +++ test/application_view_test.ml | 115 +++++++- test/journal_graph_runtime_locality_test.ml | 221 ++++++++++++++- 6 files changed, 582 insertions(+), 200 deletions(-) create mode 100644 docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md diff --git a/app/application.ml b/app/application.ml index a8976146..3cbfac2c 100644 --- a/app/application.ml +++ b/app/application.ml @@ -2120,9 +2120,7 @@ let favorites_view (fun value -> let favorite = Journal_graph_projection.favorite value in let key = favorite.membership_id in - let title = match favorite.target with - | Page _ -> Journal_model.Title.render favorite.title - | Block _ -> render_title favorite.title in + let title = render_title favorite.title in let icon = match favorite.target with | Page _ -> V.symbol ~name:"doc.text" () | Block _ -> @@ -3751,6 +3749,39 @@ let start let copy_worker_requests = Hashtbl.create 2 in let projection_request_accepted = ref (fun _ -> ()) in let projection_request_finished = ref (fun _ -> ()) in + (* Own admission once for every DB Worker API consumer. Business reducers + keep their tickets; this registry owns only delivery and scope lifetime. *) + let admitted_graph_requests = Hashtbl.create 32 in + let admitted_protocol_ids = Hashtbl.create 32 in + let dispatch_worker ?(observe_projection = false) request = + let protocol = match request with + | Graph_service.Graph_request request -> Some request + | _ -> None in + let duplicate = Option.fold ~none:false ~some:(fun request -> + Hashtbl.mem admitted_protocol_ids request.Logseq_db_worker.Protocol.request_id) protocol in + if duplicate then Worker.Full else + match Worker.send client request with + | Accepted worker_id as accepted -> + Option.iter (fun protocol -> + Hashtbl.add admitted_graph_requests worker_id protocol; + Hashtbl.add admitted_protocol_ids protocol.Logseq_db_worker.Protocol.request_id worker_id; + if observe_projection then !projection_request_accepted protocol) protocol; + accepted + | (Full | Not_ready | Stopping) as refused -> refused + in + let retire_worker_lease worker_id = + Option.iter (fun request -> + Hashtbl.remove admitted_protocol_ids request.Logseq_db_worker.Protocol.request_id) + (Hashtbl.find_opt admitted_graph_requests worker_id); + Hashtbl.remove admitted_graph_requests worker_id + in + let cancel_graph_leases () = + Hashtbl.iter (fun worker_id request -> + !projection_request_finished request.Logseq_db_worker.Protocol.request_id; + Worker.cancel client ~request_id:worker_id) admitted_graph_requests; + Hashtbl.clear admitted_graph_requests; + Hashtbl.clear admitted_protocol_ids + in let cancel_copy () = let state, _ = Journal_graph_runtime.Copy.step !copy_state Cancel in copy_state := state; @@ -3825,9 +3856,8 @@ let start ; command } in - match Worker.send client (Graph_service.Graph_request request) with + match dispatch_worker ~observe_projection:true (Graph_service.Graph_request request) with | Accepted request_id -> - !projection_request_accepted request; Hashtbl.replace copy_worker_requests request_id @@ -3886,7 +3916,7 @@ let start let media_runtime = Journal_media_runtime.create ~send:(fun ticket request -> - match Worker.send client request with + match dispatch_worker request with | Accepted id -> Option.iter (fun ticket -> Hashtbl.replace media_worker_requests id ticket) @@ -3944,7 +3974,7 @@ let start | Some _ -> state))) !set_state_ref) ~send:(fun request -> - match Worker.send client request with + match dispatch_worker request with | Worker.Accepted worker_id -> (match request with | Graph_service.Graph_request request -> @@ -4000,7 +4030,7 @@ let start let send_manager command = Effect.of_thunk (fun () -> let owner = unlock_owner !state_ref in - let result = Worker.send client (Graph_service.Client_command command) in + let result = dispatch_worker (Graph_service.Client_command command) in (match command with | Graph_service.Submit_e2ee_password _ -> (match result with @@ -4033,7 +4063,7 @@ let start Hashtbl.remove local_copy_requests id; Worker.cancel client ~request_id:id) superseded); let command = if deleting then Graph_service.Delete_local_copy request else Inspect_local_copy request in - match Worker.send client command with + match dispatch_worker command with | Worker.Accepted id -> Hashtbl.replace local_copy_requests id (request, deleting, confirm) | Full | Not_ready | Stopping -> set_state (fun state -> @@ -4092,9 +4122,8 @@ let start Journal_graph_transport.deliver ~runtime:graph_runtime ~send:(fun request -> - match Worker.send client (Graph_service.Graph_request request) with + match dispatch_worker ~observe_projection:true (Graph_service.Graph_request request) with | Accepted worker_id -> - observe_projection_request request; Hashtbl.replace graph_worker_requests worker_id request; Journal_graph_transport.Accepted | Full -> Full @@ -4102,40 +4131,38 @@ let start | Stopping -> Stopping) output in - let admission_worker_requests = Hashtbl.create 4 in - let favorites_worker_requests = Hashtbl.create 2 in - let rec run_admission_directive set_state_and_effect = function + let retire_graph_scope () = + cancel_graph_leases (); + Hashtbl.clear graph_worker_requests; + Journal_graph_runtime.reset graph_runtime + in + let rec apply_graph_responses set_state_and_effect responses = + set_state_and_effect (fun state -> + let state, effects = List.fold_left (fun (state, effects) response -> + let completion = match response.Journal_graph_runtime.payload with + | Admission_inspected { request; observation } -> + Some (request, Admission_refresh.Inspected observation) + | Admission_unavailable request -> + Some (request, Admission_refresh.Inspection_unavailable) + | _ -> None in + match completion with + | None -> Root_navigation.step state (Completed response), effects + | Some (request, result) -> + let admission_refresh, directive = Admission_refresh.complete + state.admission_refresh ~request ~result in + { state with admission_refresh }, + run_admission_directive set_state_and_effect directive :: effects) + (state, []) responses in + state, Effect.many (List.rev effects)) + and run_admission_directive set_state_and_effect = function | Admission_refresh.No_request -> Effect.ignore | Request request -> - Effect.bind + if request.graph_generation <> (!state_ref).graph_state.generation + then Effect.ignore + else Effect.bind (Effect.of_thunk (fun () -> - let output = submit (Journal_graph_request.Inspect_admission request) in - Journal_graph_transport.deliver - ~runtime:graph_runtime - ~send:(fun protocol_request -> - match - Worker.send client (Graph_service.Graph_request protocol_request) - with - | Accepted worker_request_id -> - observe_projection_request protocol_request; - Hashtbl.replace - admission_worker_requests - worker_request_id - (request, protocol_request); - Journal_graph_transport.Accepted - | Full -> Full - | Not_ready -> Not_ready - | Stopping -> Stopping) - output)) - ~f:(fun delivery -> - match delivery.Journal_graph_transport.error with - | None -> Effect.ignore - | Some _ -> - update_admission set_state_and_effect (fun state -> - Admission_refresh.complete - state.admission_refresh - ~request - ~result:Inspection_unavailable)) + deliver_output (submit (Journal_graph_request.Inspect_admission request)))) + ~f:(fun delivery -> apply_graph_responses set_state_and_effect delivery.responses) and update_admission set_state_and_effect transition = set_state_and_effect (fun state -> let admission_refresh, directive = transition state in @@ -4239,10 +4266,7 @@ let start then Effect.ignore else ( started_graph_generation := Some (graph_key, graph_state.generation); - Journal_graph_runtime.reset graph_runtime; - Hashtbl.clear graph_worker_requests; - Hashtbl.clear favorites_worker_requests; - Hashtbl.clear admission_worker_requests; + retire_graph_scope (); let current = !state_ref in let graph_info = Journal_graph_runtime.start graph_runtime in let feed_generation = current.next_request_generation in @@ -4306,6 +4330,7 @@ let start ~f:(fun delivery -> set_state (fun state -> apply_delivery_responses state delivery)))) | Graph_closed | Graph_opening | Graph_closing | Graph_failed -> + retire_graph_scope (); started_graph_generation := None; Journal_asset_runtime.shutdown asset_runtime; Effect.ignore @@ -4327,6 +4352,15 @@ let start manager.startup.graph_generation) (!state_ref).manager in if previous_owner <> Some (manager.startup.account_generation, manager.startup.graph_generation) then cancel_local_copy_checks (); + let owner_changed = previous_owner <> None + && previous_owner <> Some (manager.startup.account_generation, manager.startup.graph_generation) in + let graph_already_current = + (!state_ref).graph_state.generation = manager.startup.graph_generation + && (!state_ref).graph_state.graph_id = manager.selected_graph in + (* Graph_state and manager pushes may arrive in either order. A manager + update must not cancel work already started for its new graph scope. *) + if not manager.startup.authenticated || (owner_changed && not graph_already_current) + then (retire_graph_scope (); started_graph_generation := None); if not manager.startup.authenticated then cancel_copy (); let update = set_state (fun state -> apply_manager_state state manager_state) in let sign_out = @@ -4368,6 +4402,8 @@ let start ]) in let handle_worker_event event = + (match event with Worker.Response { request_id; _ } -> retire_worker_lease request_id + | _ -> ()); Journal_asset_runtime.pump asset_runtime; sync_media !state_ref; Journal_media_runtime.pump media_runtime; @@ -4510,15 +4546,7 @@ let start ; outcome = Worker.Completed (Graph_service.Graph_response response) ; _ } -> - let expected = - match Hashtbl.find_opt graph_worker_requests request_id with - | Some request -> Some request - | None -> - (match Hashtbl.find_opt admission_worker_requests request_id with - | Some (_, request) -> Some request - | None -> - Option.map snd (Hashtbl.find_opt favorites_worker_requests request_id)) - in + let expected = Hashtbl.find_opt graph_worker_requests request_id in let (Logseq_db_worker.Protocol.V2_response { request_id = actual; _ }) = response in let owned = match expected with @@ -4532,8 +4560,6 @@ let start | None -> false in Hashtbl.remove graph_worker_requests request_id; - Hashtbl.remove admission_worker_requests request_id; - Hashtbl.remove favorites_worker_requests request_id; let output = if owned then Journal_graph_runtime.receive graph_runtime response @@ -4549,43 +4575,13 @@ let start Effect.bind (Effect.of_thunk (fun () -> deliver_output output)) ~f:(fun delivery -> - let update = - set_state_and_effect (fun state -> - let state, effects = - List.fold_left - (fun (state, effects) response -> - let completion = - match response.Journal_graph_runtime.payload with - | Admission_inspected { request; observation } -> - Some (request, Admission_refresh.Inspected observation) - | Admission_unavailable request -> - Some (request, Admission_refresh.Inspection_unavailable) - | _ -> None - in - match completion with - | None -> Root_navigation.step state (Completed response), effects - | Some (request, result) -> - let admission_refresh, directive = - Admission_refresh.complete - state.admission_refresh - ~request - ~result - in - ( { state with admission_refresh } - , run_admission_directive set_state_and_effect directive :: effects - )) - (state, []) - delivery.responses - in - let state = - match delivery.error with - | None -> state - | Some message when Option.is_some state.feed_refresh -> - fail_feed_transport state message - | Some message -> fail_graph_transport state message - in - state, Effect.many (List.rev effects)) - in + let update = Effect.bind + (apply_graph_responses set_state_and_effect delivery.responses) + ~f:(fun () -> set_state (fun state -> + match delivery.error with + | None -> state + | Some message when Option.is_some state.feed_refresh -> fail_feed_transport state message + | Some message -> fail_graph_transport state message)) in Effect.bind update ~f:(fun () -> let refresh_after_worker_event = let (Logseq_db_worker.Protocol.V2_response { outcome; _ }) = response in @@ -4794,27 +4790,6 @@ let start forget_projection_request protocol_id; Journal_asset_runtime.reject asset_runtime ~request_id:protocol_id); Effect.ignore - | Worker.Response { request_id; outcome = Failed _ | Cancelled | Shutdown; _ } - when Hashtbl.mem favorites_worker_requests request_id -> - let request, protocol_request = Hashtbl.find favorites_worker_requests request_id in - forget_projection_request protocol_request.request_id; - Journal_graph_runtime.abandon graph_runtime protocol_request; - Hashtbl.remove favorites_worker_requests request_id; - set_state (fun state -> - favorites_event - state - (Failed (request, false, "Favorites read was interrupted. Try again."))) - | Worker.Response { request_id; outcome = Failed _ | Cancelled | Shutdown; _ } - when Hashtbl.mem admission_worker_requests request_id -> - let request, protocol_request = Hashtbl.find admission_worker_requests request_id in - forget_projection_request protocol_request.request_id; - Journal_graph_runtime.abandon graph_runtime protocol_request; - Hashtbl.remove admission_worker_requests request_id; - update_admission set_state_and_effect (fun state -> - Admission_refresh.complete - state.admission_refresh - ~request - ~result:Inspection_unavailable) | Worker.Response { request_id; outcome = (Failed _ | Cancelled | Shutdown) as outcome; _ } when Hashtbl.mem graph_worker_requests request_id -> @@ -4832,13 +4807,12 @@ let start Effect.bind (Effect.of_thunk (fun () -> deliver_output output)) ~f:(fun delivery -> - set_state (fun state -> - let state = List.fold_left apply_worker_response state delivery.responses in - match delivery.error with - | None -> state - | Some message when Option.is_some state.feed_refresh -> - fail_feed_transport state message - | Some message -> fail_graph_transport state message)) + Effect.bind (apply_graph_responses set_state_and_effect delivery.responses) + ~f:(fun () -> set_state (fun state -> + match delivery.error with + | None -> state + | Some message when Option.is_some state.feed_refresh -> fail_feed_transport state message + | Some message -> fail_graph_transport state message))) | Worker.Response { outcome = Failed error; _ } -> set_state (fun state -> let worker_error = service_error ~operation:"handleRequest" error in @@ -5100,35 +5074,8 @@ let start else Effect.bind (Effect.of_thunk (fun () -> - let output = submit (Journal_graph_request.Load_favorites request) in - Journal_graph_transport.deliver - ~runtime:graph_runtime - ~send:(fun protocol_request -> - match - Worker.send client (Graph_service.Graph_request protocol_request) - with - | Accepted worker_request_id -> - observe_projection_request protocol_request; - Hashtbl.replace - favorites_worker_requests - worker_request_id - (request, protocol_request); - Journal_graph_transport.Accepted - | Full -> Full - | Not_ready -> Not_ready - | Stopping -> Stopping) - output)) - ~f:(fun delivery -> - set_state (fun state -> - let state = - List.fold_left - (fun state response -> Root_navigation.step state (Completed response)) - state - delivery.responses - in - match delivery.error with - | None -> state - | Some message -> favorites_event state (Failed (request, false, message)))) + deliver_output (submit (Journal_graph_request.Load_favorites request)))) + ~f:(fun delivery -> apply_graph_responses set_state_and_effect delivery.responses) in Effect.bind (set_state (fun state -> @@ -5231,9 +5178,7 @@ let start then Some "The destination is no longer ready for imports." else ( match - Worker.send - client - (Graph_service.Import_asset + dispatch_worker (Graph_service.Import_asset { graph_generation = generation ; source = Journal_asset_import.to_import staged ~target }) @@ -5541,7 +5486,7 @@ let start Journal_uploads.sync current.uploads (upload_context current) in Option.iter - (fun request -> ignore (Worker.send client request : Worker.send_result)) + (fun request -> ignore (dispatch_worker request : Worker.send_result)) (Journal_uploads.retry uploads operation)) | Some (Days settings) -> Effect.of_thunk (fun () -> @@ -5785,9 +5730,7 @@ let start then Some "The destination is not ready for imports" else ( match - Worker.send - client - (Graph_service.Import_asset { graph_generation; source }) + dispatch_worker (Graph_service.Import_asset { graph_generation; source }) with | Accepted id -> Hashtbl.replace @@ -6002,6 +5945,8 @@ let start | _ -> state, Effect.ignore) | Sign_out -> cancel_copy (); + retire_graph_scope (); + started_graph_generation := None; sign_out_in_flight := true; Effect.many [ update (fun state -> let state=Root_navigation.step state Account_cleared in @@ -7138,7 +7083,7 @@ let start { app; pump; client; send_action; apply_platform; running; release_staging } in current_app := Some context; - ignore (Worker.send client Graph_service.Get_graph_state : Worker.send_result); + ignore (dispatch_worker Graph_service.Get_graph_state : Worker.send_result); Worker.on_event client (fun event -> Journal_pump.enqueue pump (fun () -> Effect.run (handle_worker_event event))); (* The worker fires the wakeup on its own domain whenever output lands; diff --git a/app/journal_graph_runtime.ml b/app/journal_graph_runtime.ml index 71c938c3..c5079e35 100644 --- a/app/journal_graph_runtime.ml +++ b/app/journal_graph_runtime.ml @@ -295,10 +295,14 @@ type failure_owner = ; block_id : string } +type request_delivery = Planned | Queued | Dispatched + type pending_operation = { operation : operation ; source_epoch : int64 ; failure_owner : failure_owner option + ; mutable background : bool + ; mutable delivery : request_delivery } type t = @@ -482,19 +486,29 @@ let request_uuid t = let uuid value = Graph.Uuid.of_string value -let make_request ?source_epoch t operation command = +let make_request ?source_epoch ?(background = false) t operation command = let source_epoch = Option.value source_epoch ~default:t.next_request in let request_id = request_uuid t in Hashtbl.replace t.pending (Graph.Uuid.to_string request_id) - { operation; source_epoch; failure_owner = None }; + { operation; source_epoch; failure_owner = None; background; delivery = Planned }; Protocol.{ api_version; request_id; command } ;; let read = make_request let mutate = make_request -let start t = read t Graph_info Protocol.V2_graph_info +(* Only a public output boundary issues a demand. Internal completions never + reserve a hydration slot or return an already dispatched request. *) +let issue_request t (request : Protocol.request) = + let key = Graph.Uuid.to_string request.request_id in + let pending = Hashtbl.find t.pending key in + pending.delivery <- Dispatched; + if pending.background then Hashtbl.replace t.hydration_active key (); + request +;; + +let start t = issue_request t (read t Graph_info Protocol.V2_graph_info) let remember_page t (page : Projection.page) = t.pages @@ -1279,7 +1293,7 @@ let submit_unowned t (request : Journal_graph_request.t) = | Error message, _ | _, Error message -> reject message) ;; -let submit t (request : Journal_graph_request.t) = +let plan_request t (request : Journal_graph_request.t) = let owner = match request with | Capture { command; _ } -> @@ -1630,27 +1644,55 @@ let registered_children t uuid = (* Keep reconciliation bursts below the worker budget without delaying foreground reads. *) let maximum_active_hydration = 4 +let favorite_request_current t pending = + match pending.operation with + | Favorite_dependency { epoch; _ } | Favorite_memberships { epoch; _ } -> + epoch = t.favorite_epoch + | _ -> true +;; + let drain_hydration t = let rec drain reversed = - if - Hashtbl.length t.hydration_active >= maximum_active_hydration - || Queue.is_empty t.hydration_queue + if Hashtbl.length t.hydration_active >= maximum_active_hydration + || Queue.is_empty t.hydration_queue then List.rev reversed else ( let request = Queue.pop t.hydration_queue in let key = Graph.Uuid.to_string request.Protocol.request_id in - if not (Hashtbl.mem t.pending key) - then drain reversed - else ( - Hashtbl.replace t.hydration_active key (); - drain (request :: reversed))) + match Hashtbl.find_opt t.pending key with + | Some pending when pending.delivery = Queued -> + if favorite_request_current t pending + then drain (issue_request t request :: reversed) + else (Hashtbl.remove t.pending key; drain reversed) + | None | Some _ -> drain reversed) in drain [] ;; -let schedule_hydration t output = - List.iter (fun request -> Queue.push request t.hydration_queue) output.requests; - { output with requests = drain_hydration t } +let background_demands t output = + List.iter (fun (request : Protocol.request) -> + match Hashtbl.find_opt t.pending (Graph.Uuid.to_string request.request_id) with + | Some pending when pending.delivery = Planned -> pending.background <- true + | None | Some _ -> ()) output.requests; + output +;; + +(* One dispatch per public transition. Delivery state makes queued/active + ownership exclusive and suppresses repeated demand identities. *) +let dispatch_output t output = + let foreground = List.filter_map (fun (request : Protocol.request) -> + match Hashtbl.find_opt t.pending (Graph.Uuid.to_string request.request_id) with + | Some pending when pending.delivery = Planned -> + if pending.background then ( + pending.delivery <- Queued; + Queue.push request t.hydration_queue; + None) + else Some (issue_request t request) + | None | Some _ -> None) output.requests in + { output with requests = foreground @ drain_hydration t } +;; + +let submit t request = dispatch_output t (plan_request t request) ;; (* Session bounds keep adversarial reference chains/bursts finite. Unresolved @@ -1695,7 +1737,7 @@ let recheck_dirty_favorite_dependency t ~uuid ~seen ~page = Hashtbl.remove t.favorite_pending key; (* The held response predates an owned change. Retire its single slot and replace it with a current point read before considering its old snapshot. *) - schedule_hydration + background_demands t { requests = Option.to_list (favorite_dependency_read ~refresh:true t ~page ~seen uuid) @@ -1842,7 +1884,7 @@ let reference_read ?(lookup = Page_lookup) ?source_epoch t block_id depth = Hashtbl.replace t.reference_pending block_id (); let uuid = Graph.Uuid.of_string block_id |> Result.get_ok in Some - (read ?source_epoch t (Reference_read { block_id; depth; lookup }) + (read ?source_epoch ~background:true t (Reference_read { block_id; depth; lookup }) (match lookup with | Page_lookup -> Protocol.V2_get_page { page = uuid; revision = None } | Outline_lookup -> @@ -2139,7 +2181,7 @@ let hydration_for_changes t ~request_generation windows = match journal_changed, t.initial_feed with | true, Some feed -> let output = - submit + plan_request t (Journal_graph_request.Load_feed { before_day = None @@ -2155,7 +2197,7 @@ let hydration_for_changes t ~request_generation windows = } | false, _ | true, None -> empty in - schedule_hydration + background_demands t { requests = point_reads @@ -2178,7 +2220,7 @@ let rehydrate_current_interests t ~request_generation ~generation = | None -> empty | Some feed -> let output = - submit + plan_request t (Journal_graph_request.Load_feed { before_day = None @@ -2257,7 +2299,7 @@ let rehydrate_current_interests t ~request_generation ~generation = read t (Changed_detail_page page) (Protocol.V2_get_page { page; revision = None })) |> List.of_seq in - schedule_hydration + background_demands t { requests = (graph_info :: journal.requests) @@ -2730,7 +2772,7 @@ let receive_response t (protocol_response : Protocol.response) = ] | _ -> [] in - schedule_hydration + background_demands t { requests = dependencies @ memberships ; responses = [ response (Favorites_loaded (request, result)) ] @@ -3205,7 +3247,7 @@ let receive_response t (protocol_response : Protocol.response) = parent in let output = - schedule_hydration + background_demands t { requests = Option.to_list request ; responses = @@ -3449,7 +3491,7 @@ let receive_response t (protocol_response : Protocol.response) = (favorite_membership_key page) (reject "Favorite membership cursor exceeds the bounded read limit.") else - schedule_hydration + background_demands t { output with requests = @@ -3709,7 +3751,10 @@ let receive t (protocol_response : Protocol.response) = let (Protocol.V2_response { request_id; _ }) = protocol_response in let key = Graph.Uuid.to_string request_id in let pending = Hashtbl.find_opt t.pending key in - let owned = Option.is_some pending in + let owned = match pending with + | Some { delivery = Dispatched; _ } -> true + | None | Some _ -> false in + if not owned then empty else ( let background = Hashtbl.mem t.hydration_active key in Hashtbl.remove t.hydration_active key; let output = @@ -3721,6 +3766,7 @@ let receive t (protocol_response : Protocol.response) = let references = match pending, protocol_response with | None, _ | Some { operation = Reference_read _; _ }, _ -> empty + | Some pending, _ when not (favorite_request_current t pending) -> empty | Some { operation = Target_summary summary; source_epoch; _ }, _ when Hashtbl.find_opt t.target_read_epochs summary.block_id <> Some source_epoch -> empty @@ -3753,7 +3799,15 @@ let receive t (protocol_response : Protocol.response) = (fun (item : Protocol.v2_child_member) -> block item.value) (Rrbvec.append_list retained items)) | V2_page_outcome (V2_present_page { page; _ }) - when Hashtbl.mem t.reference_pages (Graph.Uuid.to_string page.uuid) -> + when Hashtbl.mem t.reference_pages (Graph.Uuid.to_string page.uuid) + || (match operation with + | Detail_block { block_id; _ } -> + block_id = Graph.Uuid.to_string page.uuid + | Changed_detail_page _ -> + Hashtbl.mem t.detail_pages (Graph.Uuid.to_string page.uuid) + | _ -> false) -> + if Hashtbl.length t.reference_pages < maximum_reference_targets + then Hashtbl.replace t.reference_pages (Graph.Uuid.to_string page.uuid) (); [ ( Graph.Uuid.to_string page.uuid , if page.recycled then None else Some page.title ) ] @@ -3777,7 +3831,10 @@ let receive t (protocol_response : Protocol.response) = (fun values item -> let favorite = Projection.favorite item in match favorite.target with - | Page _ -> values + | Page id -> + if Hashtbl.length t.reference_pages < maximum_reference_targets + then Hashtbl.replace t.reference_pages id (); + (id, Some favorite.title) :: values | Block id -> (id, Some favorite.title) :: values) values result.items @@ -3875,14 +3932,8 @@ let receive t (protocol_response : Protocol.response) = values in let output = { output with responses = output.responses @ references.responses } in - if background - then - schedule_hydration t { output with requests = output.requests @ references.requests } - else if owned - then ( - List.iter (fun request -> Queue.push request t.hydration_queue) references.requests; - { output with requests = output.requests @ drain_hydration t }) - else output + if background then ignore (background_demands t output); + dispatch_output t { output with requests = output.requests @ references.requests }) ;; (* Route accepted Worker terminal failures through the same owner as protocol @@ -3930,7 +3981,7 @@ let reconcile_push t ~request_generation push = | V2_resync_required_push { generation; _ } -> rehydrate_current_interests t ~request_generation ~generation in - output + dispatch_output t output ;; module Copy = struct diff --git a/app/journal_graph_transport.ml b/app/journal_graph_transport.ml index 5236ffe4..4d5cc3b7 100644 --- a/app/journal_graph_transport.ml +++ b/app/journal_graph_transport.ml @@ -17,14 +17,29 @@ let error_message = function ;; let deliver ~runtime ~send (output : Journal_graph_runtime.output) = + let responses = ref (Rrbvec.of_list output.responses) in + let retire_unsent message requests = + let pending = Queue.create () in + List.iter (fun request -> Queue.push request pending) requests; + while not (Queue.is_empty pending) do + let request = Queue.pop pending in + let retired = Journal_graph_runtime.fail_request runtime request ~message in + responses := Rrbvec.append_list !responses retired.responses; + (* A failed completion can release queued work. It is still unsent and + must be settled, not retried against the refusing Worker. *) + List.iter (fun request -> Queue.push request pending) retired.requests + done + in let rec send_pending = function | [] -> None | request :: rest -> (match send request with | Accepted -> send_pending rest | (Full | Not_ready | Stopping) as result -> - List.iter (Journal_graph_runtime.abandon runtime) (request :: rest); - error_message result) + let message = Option.get (error_message result) in + retire_unsent message (request :: rest); + Some message) in - { responses = output.responses; error = send_pending output.requests } + let error = send_pending output.requests in + { responses = Rrbvec.to_list !responses; error } ;; diff --git a/docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md b/docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md new file mode 100644 index 00000000..4e78fbac --- /dev/null +++ b/docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md @@ -0,0 +1,39 @@ +# Unify Worker Request Scheduling + +## Problem + +A Favorite block completion drains hydration internally and the public receive boundary drains the same output again. Requests become active without reaching transport; four completed outline dependencies can strand UUID title lookups and subsequent background work. Favorite Page titles and detail Page root titles also miss shared reference discovery/rendering. + +## Proposal + +The user approved unified App-to-DB-Worker scheduling on 2026-10-17. Internal runtime transitions produce demands and mark their priority without dispatching. Public runtime entry points alone enqueue and drain once; queued and active identity sets are disjoint and deduplicated. Retain the shared four-read background bound and existing foreground admission. Transport delivers each output once and retires rejected unsent demands explicitly without retrying Worker.send. Application shares graph request admission and ownership routing instead of separate Favorites/admission/general delivery loops; Copy/media/assets retain their own state machines. Include Page titles in the existing reference observer and use the same title renderer. + +## Alternatives considered + +### Increase the background or Worker limit + +Rejected: it masks lost delivery and cannot bound arbitrary graph fan-out. + +### Add another resolver or retry loop + +Rejected: one graph-session owner already controls reference epochs and cache. Rejection must settle ownership explicitly, not silently resubmit work. + +## Acceptance criteria + +- Public Runtime regressions drain at least four Favorite block dependencies and queued references exactly once; failures, cancellation, duplicate completions, reset and epoch changes preserve capacity and fencing. +- Foreground reads remain admitted while four background demands are outstanding; only explicit missing Page results trigger outline fallback. +- Transport refusal releases all unsent ownership and reports failure without hidden send retries. +- Favorite Page titles and detail Page roots discover references; display uses shared lookup and creates no reference link. +- Existing targeted tests and full build/tests pass; validate controlled fixtures and the existing A135 Lambda RTC read-only flow with identified artifacts. +- Do not modify Dune, protected spec interfaces, real graph data or the phone. Any required protected interface change needs a precise separately approved diff. +- Open a formal PR, run an independent review session using journal-code-review, address valid findings and follow CI. Do not merge. + +## Risks + +- Priority inheritance must preserve initial foreground feed loads and bounded background refreshes. +- Worker rejection can produce more failure completions; settlement must remain bounded and must not send retries. +- Experimental host-restamped Simulator artifacts retain cross_abi_validated=false and require fresh source provenance after changes. + +## Questions + +- None. The user explicitly approved this scope; no protected interface change is currently proposed. diff --git a/test/application_view_test.ml b/test/application_view_test.ml index 428bb901..105ef608 100644 --- a/test/application_view_test.ml +++ b/test/application_view_test.ml @@ -4397,6 +4397,119 @@ let test_detail_page_property_value_presentation () = [true;false] ;; +(* The public Favorites mount has no shared-lookup input. Exercise the narrow + Application ownership/renderer boundary with a real Worker and correct page + data; Runtime discovery itself is covered by its pure owner regressions. *) +let test_favorite_page_uses_shared_reference_lookup () = + let module W = Logseq_db_worker_lui.Journal_worker in + let module P = Logseq_db_worker.Protocol in + let module G = Logseq_db_types.Graph_types in + let uuid n = G.Uuid.of_string + (Printf.sprintf "86000000-0000-4000-8000-%012d" n) |> Result.get_ok in + let token = "[[" ^ G.Uuid.to_string (uuid 3) ^ "]]" in + let page : G.page = + { uuid = uuid 1; name = "favorite"; title = "Page " ^ token; kind = Ordinary_page + ; created_at_ms = 0L; updated_at_ms = 0L; recycled = false; tags = []; properties = [] } in + let block : G.block = + { uuid = uuid 2; title = "Block " ^ token; parent = page.uuid; page = page.uuid + ; order = "a"; created_at_ms = 0L; updated_at_ms = 0L; refs = []; tags = []; properties = [] } in + let manager : Service.state = + { snapshot = + { sync_phase = Current; catalog = []; selected_graph = Some (uuid 9) + ; applied_server_t = Some (server_cursor_of_int 0); timeline_presentation_pending = false + ; startup = { authenticated = true; catalog_loading = false; awaiting_selection = false + ; restoring_local = false; bootstrapping = false; awaiting_e2ee_password = false + ; failure = None; account_generation = 1; graph_generation = 1; presentation_generation = 1 } + ; last_error = None; local_deletion = None } + ; diagnostics = { groups = [] } } in + let target_reads = Atomic.make 0 in + let service = W.Service.create ~push_topic_count:6 ~concurrency:(Concurrent { max_in_flight = 2 }) + ~init:(fun context _ -> + W.Session_context.emit context ~topic:Service.manager_topic (Service.Client_state_changed manager); + Ok ()) + ~handle:(fun _ () -> function + | Service.Get_graph_state -> Ok (Service.Graph_state + { generation = 1; graph_id = Some (uuid 9); phase = Graph_open; error = None }) + | Client_command _ -> Ok Service.Client_command_completed + | Graph_request request -> + let open P in + let outcome = match request.command with + | V2_graph_info -> V2_graph_info_outcome + { graph_uuid = uuid 9; graph_name = "Fixture"; schema = { major = 65; minor = 33 } + ; admission_facts = []; journal_title_format = None; generation = "g"; projection_revision = "p" + ; limits = { response_budget_bytes = maximum_response_bytes; outbox_max_records = 4096 + ; outbox_max_bytes = 8388608; change_max_items = 4096; change_max_bytes = 1048576 + ; dispatcher_capacity = 256; wire_batch_max_bytes = 262144 } } + | V2_list_journals _ -> V2_journals_outcome { items = []; next_cursor = None } + | V2_list_favorites _ -> V2_favorites_outcome + { favorites_page = None; generation = "g"; projection_revision = "p"; next_cursor = None + ; items = + [{ membership_uuid = uuid 10; membership_order = "a"; membership_revision = "m" + ; target = V2_favorite_page { uuid = page.uuid; title = page.title; revision = "p" } } + ;{ membership_uuid = uuid 11; membership_order = "b"; membership_revision = "m" + ; target = V2_favorite_block + { uuid = block.uuid; title = block.title; task_status = None; revision = "b" } }] } + | V2_get_page { page = requested; _ } -> + if G.Uuid.equal requested (uuid 3) then Atomic.incr target_reads; + V2_page_outcome (V2_present_page + { page = if G.Uuid.equal requested page.uuid then page + else { page with uuid = requested; title = "Resolved"; name = "resolved" } + ; revision = "p" }) + | V2_get_outline_block { block = requested; _ } when G.Uuid.equal requested block.uuid -> + V2_block_outcome (V2_present_block + { value = { block; task_status = None; rendered_page_title = page.title; tag_titles = [] } + ; revision = "b" }) + | V2_list_assets _ -> V2_assets_outcome + { generation = "g"; projection_revision = "p"; items = []; next_cursor = None } + | _ -> V2_failed { code = "InvalidRequest"; message = "Unused fixture operation" } + in + Ok (Service.Graph_response (V2_response + { api_version; request_id = request.request_id; outcome })) + | _ -> Error "Unused fixture service operation") + ~shutdown:(fun () -> ()) () in + let client = ref None in + let hooks = Application.For_testing.app_with_service + ~on_client:(fun c -> client := Some c) + ~calendar_sampler:(Journal_calendar.Sampler.create + ~clock:(fun () -> 1_788_192_000.) ~localtime:Unix.gmtime ()) service in + let props = Hashtbl.create 64 in + let track = track_json_wire_teardown (Hashtbl.remove props) in + let consume encoded = if encoded <> "" then ( + let open Yojson.Safe.Util in + Yojson.Safe.from_string encoded |> member "ops" |> to_list |> List.iter (fun op -> + track op; + match member "op" op |> to_string with + | "set-prop" | "set-extension-prop" -> + let id = member "id" op |> to_int in + let key = member "property" op |> to_string in + let old = Hashtbl.find_opt props id |> Option.value ~default:[] in + Hashtbl.replace props id ((key, member "value" op) :: List.remove_assoc key old) + | _ -> ())) in + let find key text = Hashtbl.fold (fun id pairs found -> + if List.assoc_opt key pairs = Some (`String text) then Some id else found) props None in + let wait label predicate = + let deadline = Unix.gettimeofday () +. 3. in + while not (predicate ()) && Unix.gettimeofday () < deadline do + consume (hooks.pump ()); Unix.sleepf 0.001 + done; + Alcotest.(check bool) label true (predicate ()) in + Fun.protect ~finally:(fun () -> + ignore (hooks.dispose ()); Option.iter Logseq_db_worker_lui.Journal_worker_runtime.stop !client) + (fun () -> + let startup = Logseq_db_worker.Config.create + ~application_support_directory:"/tmp/journal-favorite-page-reference-fixture" + ~target:(Managed_sync { base_url = "https://example.invalid" }) + ~compatibility_profile:Logseq_65_33_or_newer + ~response_budget_bytes:P.maximum_response_bytes ~default_page_size:P.default_page_size + |> Result.get_ok |> Journal_startup.encode |> Result.get_ok |> Bytes.to_string in + consume (hooks.init 2 2 startup); + wait "Favorites navigation mounts" (fun () -> Option.is_some (find "accessibility-label" "Favorites")); + consume (hooks.dispatch (Lui_protocol.Press (Option.get (find "accessibility-label" "Favorites")))); + wait "block proves shared lookup is populated" (fun () -> Option.is_some (find "text" "Block Resolved")); + wait "Favorite Page uses the same populated lookup" (fun () -> Option.is_some (find "text" "Page Resolved")); + Alcotest.(check int) "shared target is queried once" 1 (Atomic.get target_reads)) +;; + let test_favorites_compact_shared_titles () = let module G=Logseq_db_types.Graph_types in let module P=Logseq_db_worker.Protocol in @@ -5515,7 +5628,7 @@ let () = ; ( "status list", List.map (fun scenario -> Alcotest.test_case scenario `Quick (fun () -> run_favorites_native_visibility ~status_scenario:scenario ())) ["expanded"; "none"; "now"; "waiting"; "later"; "select"; "clear"; "dismiss"; "failure"; "duplicate"]) - ; ("Favorites compact title",[Alcotest.test_case "page property values keep name without duplication" `Quick test_detail_page_property_value_presentation; Alcotest.test_case "root Capture only in Journals" `Quick test_root_capture_placement; Alcotest.test_case "LUI titles icons and no arrow" `Quick test_favorites_compact_shared_titles; Alcotest.test_case "page outline click and native Back" `Quick (fun () -> run_favorites_native_visibility ~check_favorite_targets:true ())]) + ; ("Favorites compact title",[Alcotest.test_case "Favorite Page uses shared reference lookup" `Quick test_favorite_page_uses_shared_reference_lookup; Alcotest.test_case "page property values keep name without duplication" `Quick test_detail_page_property_value_presentation; Alcotest.test_case "root Capture only in Journals" `Quick test_root_capture_placement; Alcotest.test_case "LUI titles icons and no arrow" `Quick test_favorites_compact_shared_titles; Alcotest.test_case "page outline click and native Back" `Quick (fun () -> run_favorites_native_visibility ~check_favorite_targets:true ())]) ; ( "graph picker", [ Alcotest.test_case "refresh failure replaces old presence failure" `Quick (test_graph_picker_controls `Refresh_after_presence_failure); Alcotest.test_case "catalog refresh failure is visible" `Quick (test_graph_picker_controls `Refresh_failure); diff --git a/test/journal_graph_runtime_locality_test.ml b/test/journal_graph_runtime_locality_test.ml index 14b2f765..ee33f076 100644 --- a/test/journal_graph_runtime_locality_test.ml +++ b/test/journal_graph_runtime_locality_test.ml @@ -5569,6 +5569,216 @@ let test_favorites_outline_and_missing_reference () = (List.assoc_opt target_id (reference_updates out) |> Option.join)) [false;true] ;; +(* The scheduler is owned by Runtime, so use only its public demands and + completions. Every Present value below describes the requested entity. *) +let scheduling_uuid family n = + uuid (Printf.sprintf "a8000000-0000-4000-%04d-%012d" family n) +;; + +let scheduling_block n : Protocol.v2_block_record = + let parent = scheduling_uuid 8000 n in + { record with + block = + { block with + uuid = scheduling_uuid 9000 n + ; parent + ; page = parent + ; title = if n = 0 then "Favorite " ^ reference else "Favorite block" + } + } +;; + +let scheduling_favorites runtime count generation = + let request : Journal_graph_request.favorites_request = + { graph_generation = 1; request_generation = generation; limit = 50; cursor = None } + in + let read = Runtime.submit runtime (Load_favorites request) + |> fun out -> only "Favorite list demand" out.requests in + Runtime.receive runtime (response read (Protocol.V2_favorites_outcome + { favorites_page = None; generation = "g"; projection_revision = "p" + ; next_cursor = None + ; items = List.init count (fun n -> + let value = scheduling_block n in + { Protocol.membership_uuid = scheduling_uuid 7000 n + ; membership_order = string_of_int n; membership_revision = "m" + ; target = V2_favorite_block + { uuid = value.block.uuid; title = value.block.title + ; task_status = None; revision = "b" } }) })) +;; + +let scheduling_reply (request : Protocol.request) = + match request.command with + | V2_get_outline_block { block = requested; _ } -> + let n = List.find (fun n -> Graph.Uuid.equal requested (scheduling_uuid 9000 n)) + (List.init 12 Fun.id) in + response request (Protocol.V2_block_outcome + (V2_present_block { value = scheduling_block n; revision = "b" })) + | V2_get_page { page = requested; _ } -> + response request (Protocol.V2_page_outcome (V2_present_page + { page = { page with uuid = requested; kind = Ordinary_page + ; title = if Graph.Uuid.equal requested target_uuid then "Resolved" else "Parent" } + ; revision = "p" })) + | _ -> Alcotest.fail "unexpected scheduling demand" +;; + +let drain_scheduling ?(cancel_count = 0) runtime output = + let issued = Hashtbl.create 32 in + let target_reads = ref 0 and updates = ref [] in + let collect (output : Runtime.output) = + updates := reference_updates output @ !updates; + List.iter (fun (request : Protocol.request) -> + let key = Graph.Uuid.to_string request.request_id in + Alcotest.(check bool) "one delivery per request ID" false (Hashtbl.mem issued key); + Hashtbl.add issued key (); + if request.command = V2_get_page { page = target_uuid; revision = None } + then incr target_reads) output.requests + in + collect output; + let rec loop completed outstanding = + check_hydration_bound outstanding; + match outstanding with + | [] -> () + | request :: rest -> + let next = if completed < cancel_count then + Runtime.fail_request runtime request ~message:"Worker cancelled" + else Runtime.receive runtime (scheduling_reply request) in + let duplicate = Runtime.receive runtime (scheduling_reply request) in + Alcotest.(check int) "late or repeated completion cannot dispatch" 0 + (List.length duplicate.requests); + Alcotest.(check int) "late or repeated completion cannot publish" 0 + (List.length duplicate.responses); + collect next; + loop (completed + 1) (rest @ next.requests) + in + loop 0 output.requests; + Alcotest.(check int) "queued target page is delivered exactly once" 1 !target_reads; + Alcotest.(check (option string)) "target response reaches shared cache" + (Some "Resolved") (List.assoc_opt target_id !updates |> Option.join); + let fresh = scheduling_favorites runtime 4 902L in + Alcotest.(check int) "all four slots are free after draining" 4 + (List.length fresh.requests); + Alcotest.(check (list string)) "no earlier queued identity leaks into a fresh window" + (List.init 4 (fun n -> Graph.Uuid.to_string (scheduling_uuid 9000 n)) |> List.sort String.compare) + (List.map (fun (request : Protocol.request) -> match request.command with + | V2_get_outline_block { block; _ } -> Graph.Uuid.to_string block + | _ -> Alcotest.fail "fresh window returned a leftover request") fresh.requests + |> List.sort String.compare) +;; + +let test_favorite_dependencies_drain_once () = + List.iter (fun count -> + let runtime = Runtime.create () in + set_calendar runtime; + drain_scheduling runtime (scheduling_favorites runtime count 900L)) [4; 8] +;; + +let test_favorite_cancel_releases_slot () = + List.iter (fun cancel_count -> + let runtime = Runtime.create () in + set_calendar runtime; + drain_scheduling ~cancel_count runtime (scheduling_favorites runtime 8 900L)) [1; 4] +;; + +let test_favorite_scope_discards_queued_reads () = + let runtime = Runtime.create () in + set_calendar runtime; + let old = scheduling_favorites runtime 8 900L in + Runtime.reset runtime; + List.iter (fun request -> + let late = Runtime.receive runtime (scheduling_reply request) in + Alcotest.(check int) "old scope cannot dispatch queued reads" 0 + (List.length late.requests); + Alcotest.(check int) "old scope cannot publish sources" 0 + (List.length late.responses)) old.requests; + drain_scheduling runtime (scheduling_favorites runtime 8 901L) +;; + +let test_favorite_epoch_preserves_new_queue () = + let runtime = Runtime.create () in + set_calendar runtime; + let old = scheduling_favorites runtime 8 900L in + let current = scheduling_favorites runtime 8 901L in + Alcotest.(check int) "new epoch shares the outstanding window" 0 + (List.length current.requests); + let resumed = List.fold_left (fun (out : Runtime.output) request -> + let next = Runtime.receive runtime (scheduling_reply request) in + { Runtime.requests = out.requests @ next.requests + ; responses = out.responses @ next.responses }) current old.requests in + drain_scheduling runtime resumed +;; + +(* Worker admission belongs to transport. Test only that missing boundary, + with a real Runtime output; do not duplicate DB/effect-runner coverage. *) +let test_rejected_delivery_retires_unsent_queue () = + List.iter (fun refusal -> + let runtime = Runtime.create () in + set_calendar runtime; + let output = scheduling_favorites runtime 8 900L in + let attempts = ref 0 in + let delivery = Journal_graph_transport.deliver ~runtime + ~send:(fun _ -> incr attempts; refusal) output in + Alcotest.(check int) "refusal is never retried" 1 !attempts; + Alcotest.(check bool) "refusal remains explicit" true (Option.is_some delivery.error); + List.iter (fun request -> + let late = Runtime.receive runtime (scheduling_reply request) in + Alcotest.(check int) "rejected request cannot publish late response" 0 + (List.length late.responses)) output.requests; + drain_scheduling runtime (scheduling_favorites runtime 8 901L)) + Journal_graph_transport.[Full; Not_ready; Stopping] +;; + +let test_favorite_page_title_discovers_reference () = + let runtime = Runtime.create () in + set_calendar runtime; + let request : Journal_graph_request.favorites_request = + { graph_generation = 1; request_generation = 903L; limit = 50; cursor = None } in + let read = Runtime.submit runtime (Load_favorites request) + |> fun out -> only "Favorite page list" out.requests in + let out = Runtime.receive runtime (response read (Protocol.V2_favorites_outcome + { favorites_page = None; generation = "g"; projection_revision = "p" + ; next_cursor = None + ; items = [{ Protocol.membership_uuid; membership_order = "a"; membership_revision = "m" + ; target = V2_favorite_page { uuid = page_uuid; title = reference; revision = "p" } }] })) in + let reference_read = out.requests |> List.filter (fun (r : Protocol.request) -> + r.command = V2_get_page { page = target_uuid; revision = None }) + |> only "Favorite page title demands page-first reference" in + let resolved = Runtime.receive runtime + (reference_page_response reference_read Ordinary_page "Resolved") in + Alcotest.(check (option string)) "Favorite page source is accepted" (Some "Resolved") + (List.assoc_opt target_id (reference_updates resolved) |> Option.join) +;; + +let test_page_detail_root_title_discovers_reference () = + let runtime = Runtime.create () in + set_calendar runtime; + let root = Runtime.submit runtime (Load_detail + { block_id = Graph.Uuid.to_string page_uuid; after = None + ; limit = 50; request_generation = 904L }) + |> fun out -> only "detail root" out.requests in + let page_read = Runtime.receive runtime (response root (Protocol.V2_block_outcome + (V2_missing_block { uuid = page_uuid; revision = "no-outline" }))) + |> fun out -> only "detail page fallback" out.requests in + let out = Runtime.receive runtime (response page_read (Protocol.V2_page_outcome + (V2_present_page { page = { page with kind = Ordinary_page; title = reference } + ; revision = "p" }))) in + let children = out.requests |> List.find (fun (r : Protocol.request) -> + match r.command with V2_get_children _ -> true | _ -> false) in + let loaded = Runtime.receive runtime (response children (Protocol.V2_children_outcome + { parent = page_uuid; revision_scope = V2_children_revision page_uuid + ; scope_revision = "c"; items = []; next_cursor = None })) in + let refs = out.requests @ loaded.requests |> List.filter (fun (r : Protocol.request) -> + r.command = V2_get_page { page = target_uuid; revision = None }) in + let ref_read = only "empty page detail root demands reference" refs in + let resolved = Runtime.receive runtime + (reference_page_response ref_read Ordinary_page "Resolved") in + let lookup id = List.assoc_opt id (reference_updates resolved) |> Option.join in + let runs = Journal_model.Title.render ~lookup reference in + Alcotest.(check string) "page root displays the shared result" "Resolved" + (Journal_model.Title.plain runs); + Alcotest.(check bool) "reference expansion is not clickable" true + (List.for_all (fun (run : Journal_model.Title.run) -> run.url = None) runs) +;; + let test_copy_page_root () = List.iter (fun populated -> let state,effects=copy_start page_uuid in @@ -5590,7 +5800,16 @@ let test_copy_page_root () = let () = Alcotest.run "journal graph runtime locality" - [ ("Favorites page detail", [Alcotest.test_case "page property refresh discovers references" `Quick test_page_property_refresh_discovers_references; Alcotest.test_case "page root and scoped append" `Quick test_page_detail_root_and_append; Alcotest.test_case "copy page root and empty page" `Quick test_copy_page_root; Alcotest.test_case "recycled page is unavailable" `Quick test_recycled_page_root_is_removed; Alcotest.test_case "page point-change identity" `Quick test_page_root_block_change_uses_page_identity; Alcotest.test_case "page root refresh and resync" `Quick test_page_root_refresh_and_resync; Alcotest.test_case "Favorites share page-first reference cache" `Quick test_favorites_share_reference_cache; Alcotest.test_case "Favorites outline and absent references" `Quick test_favorites_outline_and_missing_reference]); ( "review reference ownership", [ + [ ("unified request scheduling", [ + Alcotest.test_case "four and eight Favorites drain exactly once" `Quick test_favorite_dependencies_drain_once; + Alcotest.test_case "cancelled dependency releases capacity" `Quick test_favorite_cancel_releases_slot; + Alcotest.test_case "scope reset discards old queue" `Quick test_favorite_scope_discards_queued_reads; + Alcotest.test_case "Favorite epoch preserves new queue" `Quick test_favorite_epoch_preserves_new_queue; + Alcotest.test_case "refused delivery settles all unsent demands" `Quick test_rejected_delivery_retires_unsent_queue; + Alcotest.test_case "Favorite page discovers title reference" `Quick test_favorite_page_title_discovers_reference; + Alcotest.test_case "empty page detail discovers root reference" `Quick test_page_detail_root_title_discovers_reference + ]); + ("Favorites page detail", [Alcotest.test_case "page property refresh discovers references" `Quick test_page_property_refresh_discovers_references; Alcotest.test_case "page root and scoped append" `Quick test_page_detail_root_and_append; Alcotest.test_case "copy page root and empty page" `Quick test_copy_page_root; Alcotest.test_case "recycled page is unavailable" `Quick test_recycled_page_root_is_removed; Alcotest.test_case "page point-change identity" `Quick test_page_root_block_change_uses_page_identity; Alcotest.test_case "page root refresh and resync" `Quick test_page_root_refresh_and_resync; Alcotest.test_case "Favorites share page-first reference cache" `Quick test_favorites_share_reference_cache; Alcotest.test_case "Favorites outline and absent references" `Quick test_favorites_outline_and_missing_reference]); ( "review reference ownership", [ Alcotest.test_case "typed metadata retains newer fence" `Quick test_typed_metadata_preserves_newer_reference_fence; Alcotest.test_case "dirty abandon remains retryable" `Quick test_dirty_abandon_keeps_reference_retryable; Alcotest.test_case "failed lookup retains epoch fence" `Quick test_reference_failed_read_retains_epoch_fence; From 6ab0ead1818a1cf61b5af885e0a43950727046d8 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 17 Oct 2026 14:03:18 +0800 Subject: [PATCH 2/5] Verify ordinary XMLm native inputs without forced retention --- .../journal-build/references/cache-and-ci.md | 2 +- .../references/native-provenance.md | 64 ++++ .../journal-build/references/targets.md | 2 +- .../journal-build/scripts/journal_build.py | 297 +++++++++++++++--- .../scripts/test_journal_build.py | 297 ++++++++++++++++++ ...26-10-17-allow-unused-xmlm-native-input.md | 39 +++ 6 files changed, 649 insertions(+), 52 deletions(-) create mode 100644 docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md diff --git a/.agents/skills/journal-build/references/cache-and-ci.md b/.agents/skills/journal-build/references/cache-and-ci.md index 0d1e0223..28eba1b4 100644 --- a/.agents/skills/journal-build/references/cache-and-ci.md +++ b/.agents/skills/journal-build/references/cache-and-ci.md @@ -3,7 +3,7 @@ Identity includes source, actual dependency ABI inputs, toolchain/SDK, flags and target. Identical inputs reuse incremental state; changed inputs select another namespace. Every run leaves external phase logs and `run-*/manifest.json`. The cache has a nonblocking lock; native Dune builds also hold the switch read lock. A busy lock belongs to its active owner: coordinate or wait. Preserve existing output/rollback apps and choose a new `--app-dir`; nothing is cleared automatically. -For failure, read `failed_phase`, `error` and that phase's log. Correct target/environment or select matching prepared inputs. Source/configuration/pin/current-input mismatch requires matching provenance; a historical archive change alone can use the validated receipt contract above. Missing mldoc/XMLm retention requires linker-rule review; protected Dune/spec/lock edits still follow the repository's authorization rules. Legacy object-injecting Dune aliases remain refused; probe helpers use the isolated mode above. +For failure, read `failed_phase`, `error` and that phase's log. Correct target/environment or select matching prepared inputs. Source/configuration/pin/current-input mismatch requires matching provenance; a historical archive change alone can use the validated receipt contract above. Missing runtime/mldoc or unresolved XMLm references require linker-rule review. Schema 2 permits an unused XMLm unit to be omitted while retaining strict locked input identity; schema 1 keeps its original force-link checks. Protected Dune/spec/lock edits still follow the repository's authorization rules. Legacy object-injecting Dune aliases remain refused; probe helpers use the isolated mode above. ## Reuse dependency and build state diff --git a/.agents/skills/journal-build/references/native-provenance.md b/.agents/skills/journal-build/references/native-provenance.md index 4dd60073..4a09fe7d 100644 --- a/.agents/skills/journal-build/references/native-provenance.md +++ b/.agents/skills/journal-build/references/native-provenance.md @@ -1,5 +1,69 @@ # Experimental native provenance +Input identity and retained code are separate checks. Journal's Markdown path +requires the OCaml runtime and mldoc. XMLm is a locked dependency used by mldoc's +OPML units; an unused XMLm unit may be removed by the OCaml linker. Any unresolved +XMLm reference is rejected. No application anchor or global `-linkall` is needed. + +## Ordinary archive inputs (schema 2) + +New producers use `schema_version: 2`, with hash-bound `source` and `link` records +and optional `validated_build`. They do not supply `xmlm_force_link`. Source, +compiler, pins, target C and complete-object identities retain the checks below. + +The link record adds an absolute `working_directory` and records the actual, +expanded `-output-complete-obj` argv, including its output. Every explicit native +input and its `.a`/`.o` companion must occur by absolute path in `direct_native_input_sha256`. +Both original mldoc and XMLm archives must occur in that argv with the recorded +archive hash; response-file commands and output-as-input identities are refused. +Recorded argv is inspected as data, never executed by the importer. + +Schema 2 supports the object-only Dune link form: the fixed code-generation +switches, `-w`/`-warn-error` values (including `@` warning sets), `-I`, `-o`, and +`-cclib` with one unquoted token naming an explicit bound archive/object or `-L` +directory; whitespace, quotes and backslashes in that value are refused because +OCaml can split it into additional linker arguments. Source files, +plugins, response files, `-l` providers, `-ccopt` and other unrecognized options +are refused rather than treated as proved inputs. Relative argv paths resolve +through the recorded working directory for every later archive/stub check. +Companion and dependency hash claims must agree; merging records cannot replace +a conflicting actual-input digest. + +Recorded absolute input names use lexical normalization without resolving +symlinks and cannot contain `..`. First import hashes the current files through +those names. Receipt reuse retains the same recorded names even when historical +installation symlinks or their targets later disappear or change. + +Each `new_dependency_provenance` entry for mldoc/XMLm retains `pinned_commit`, +`installed_archive`, archive/static archive hashes and installed source hashes. +It additionally supplies: + +```json +{ + "source_checkout": "/absolute/clean/pinned/checkout", + "original_unit": {"cmx_sha256": "SHA256", "object_sha256": "SHA256"}, + "installed_sources_matched_to_pinned_git": { + "xmlm.ml": {"sha256": "SHA256", "git_path": "src/xmlm.ml"} + } +} +``` + +First import checks the clean checkout's exact pinned commit, hashes its raw Git +source blobs against the installed source records, and verifies the original +cmx and archive object member. Installation paths and Git paths remain inside +their respective roots. The immutable complete object's defined and undefined +symbols are inspected separately. `ocaml_units` records actual booleans for +`runtime`, `mldoc` and `xmlm`; runtime/mldoc must be present, XMLm may be absent. + +A schema-2 receipt records `ocaml_units` in its imported dependencies and final +`production_link`, without the legacy `runtime_mldoc_xmlm_retained` flag. Those +observations must agree with each other and the still-present fixed host object. +Original source/archive/compiler input hashes may become historical only after +that successful hash-bound receipt, while current native source, object, runtime +headers, original evidence and SDK/signing checks remain live. A schema-1 receipt +cannot authorize schema 2, or vice versa. + +## Legacy single-unit force link (schema 1) The envelope references original producer records by SHA-256; paths are relative to the envelope, while producer records retain their own explicit paths: diff --git a/.agents/skills/journal-build/references/targets.md b/.agents/skills/journal-build/references/targets.md index 28b9b583..946da4d8 100644 --- a/.agents/skills/journal-build/references/targets.md +++ b/.agents/skills/journal-build/references/targets.md @@ -32,6 +32,6 @@ bash .agents/skills/journal-build/scripts/build_journal_apple.sh ios-device --ex ## Accept and hand off -Only `status: built` confirms an artifact. Check recorded revisions/dirty hashes, target/SDK/flags, bundle/executable hashes, version and signing identity. Resources/icons precede signing; device profile/certificate/team/entitlements and strict signature are checked. Current GMP archives are rehashed before native macOS acceptance. Experimental builds additionally check Swift object platforms, fresh target C link-map providers and runtime/mldoc/XMLm retention. Listing a `.cmxa` alone does not establish retention. +Only `status: built` confirms an artifact. Check recorded revisions/dirty hashes, target/SDK/flags, bundle/executable hashes, version and signing identity. Resources/icons precede signing; device profile/certificate/team/entitlements and strict signature are checked. Current GMP archives are rehashed before native macOS acceptance. Experimental builds additionally check Swift object platforms, fresh target C link-map providers and runtime/mldoc retention. Record XMLm's actual retention separately from its locked input identity: schema 2 allows its unused unit to be omitted and rejects unresolved references; legacy schema 1 still requires its force-link proof and retention. Listing a `.cmxa` alone does not establish retention. `preflight-passed` and `link-only` are not runtime acceptance. Preserve `cross_abi_validated=false` for host-restamped objects and state which compilation was reused. Carry the manifest with the App; installation and phone/graph operations are separate from build. For builder changes, use `PYTHONDONTWRITEBYTECODE=1 python3 .agents/skills/journal-build/scripts/test_journal_build.py` and the existing icon test as appropriate. diff --git a/.agents/skills/journal-build/scripts/journal_build.py b/.agents/skills/journal-build/scripts/journal_build.py index 7b0d062d..ced7adc8 100644 --- a/.agents/skills/journal-build/scripts/journal_build.py +++ b/.agents/skills/journal-build/scripts/journal_build.py @@ -250,14 +250,158 @@ def dependencies(self): self.report["dependencies"]["installed_lib_hashes"] = files(self.prefix / "lib", native_only=True) return self.report["dependencies"] + @staticmethod + def valid_ocaml_units(units): + return (isinstance(units, dict) and set(units) == {"runtime", "mldoc", "xmlm"} + and all(type(value) is bool for value in units.values()) + and units["runtime"] and units["mldoc"]) + + def verify_ocaml_units(self, product, *, require_xmlm): + symbols = self.run("nm", "-j", "-U", product) + def present(unit): + return bool(re.search(r"^_caml" + unit + r"(?:__|_|\$|$)", symbols, re.MULTILINE)) + units = {"runtime": "_caml_startup" in symbols.splitlines(), + "mldoc": present("Mldoc"), "xmlm": present("Xmlm")} + if not self.valid_ocaml_units(units) or (require_xmlm and not units["xmlm"]): + raise ValueError("native object must retain OCaml runtime and mldoc" + + (" and legacy XMLm" if require_xmlm else "")) + undefined = self.run("nm", "-j", "-u", product) + if re.search(r"^_camlXmlm(?:__|_|\$|$)", undefined, re.MULTILINE): + raise ValueError("native object contains unresolved XMLm references") + return units + + @staticmethod + def native_input_name(name): + path = Path(name) + if not path.is_absolute() or ".." in path.parts: + raise ValueError(f"native input identity needs an absolute path without parent traversal: {name}") + # The recorded pathname is historical identity. Resolving a symlink + # again after its removal/retargeting would change that identity. + return os.path.normpath(str(path)) + + @staticmethod + def bind_native_inputs(inputs, additions): + for name, digest in additions.items(): + path = Build.native_input_name(name) + if path in inputs and inputs[path] != digest: + raise ValueError(f"conflicting native input hash identity: {path}") + inputs[path] = digest + + def ordinary_native_inputs(self, pre, link): + """Bind the expanded producer argv to hashed real inputs, without executing it.""" + command = link["command"] + cwd = Path(link["working_directory"]) + if (not command or not all(isinstance(arg, str) for arg in command) + or not cwd.is_absolute() or not Path(command[0]).is_absolute() + or "-output-complete-obj" not in command or command.count("-o") != 1): + raise ValueError("ordinary producer needs an expanded complete-object command and absolute compiler/cwd") + def absolute(arg): + path = Path(arg) + return self.native_input_name(path if path.is_absolute() else cwd / path) + output_index = command.index("-o") + 1 + if output_index == len(command) or absolute(command[output_index]) != self.native_input_name(pre["production_native_object"]): + raise ValueError("native command output differs from the producer object") + recorded = link["direct_native_input_sha256"] + if (any(not Path(path).is_absolute() for path in recorded) + or any(not re.fullmatch(r"[0-9a-f]{64}", digest) for digest in recorded.values())): + raise ValueError("native input digest is invalid") + canonical = {absolute(path): digest for path, digest in recorded.items()} + if len(canonical) != len(recorded) or absolute(command[output_index]) in canonical: + raise ValueError("native input identities alias each other or the output") + actual, normalized = set(), [command[0]] + def native_input(arg, suffixes): + suffix = Path(arg).suffix + if arg.startswith(("@", "-")) or suffix not in suffixes: + raise ValueError(f"unsupported native input argument: {arg}") + path = absolute(arg) + actual.add(path) + required = {path} + if suffix in {".cmxa", ".cmx"}: + required.add(str(Path(path).with_suffix(".a" if suffix == ".cmxa" else ".o"))) + if not required.issubset(canonical): + raise ValueError(f"actual native link input is not hash-bound: {arg}") + return path + # Object-only Dune argv. Unknown compiler/linker flags cannot introduce + # source, plugins, response files or unrecorded -l providers. + switches = {"-output-complete-obj", "-g", "-strict-sequence", "-strict-formats", + "-short-paths", "-keep-locs"} + index = 1 + while index < len(command): + arg = command[index] + if arg in switches: + normalized.append(arg); index += 1 + elif arg in {"-w", "-warn-error", "-I", "-o", "-cclib"}: + if index + 1 == len(command): + raise ValueError(f"native argument needs a value: {arg}") + value = command[index + 1] + if arg in {"-I", "-o"}: + value = absolute(value) + elif arg == "-cclib": + if re.search(r"[\s'\"\\]", value): + raise ValueError("native -cclib input must be one unquoted token") + if value.startswith("-L") and len(value) > 2: + value = "-L" + absolute(value[2:]) + else: + value = native_input(value, {".a", ".o"}) + normalized.extend([arg, value]); index += 2 + elif arg.startswith("-"): + raise ValueError(f"unsupported native compiler/linker argument: {arg}") + else: + normalized.append(native_input(arg, {".cmxa", ".cmx", ".a", ".o"})); index += 1 + self.ordinary_command = normalized + for name in ("mldoc", "xmlm"): + dependency = link["new_dependency_provenance"][name] + archive = absolute(dependency["installed_archive"]) + if archive not in actual or canonical.get(archive) != dependency["archive_sha256"]: + raise ValueError(f"original {name} archive is not the declared actual link input") + if canonical.get(str(Path(archive).with_suffix(".a"))) != dependency["static_archive_sha256"]: + raise ValueError(f"conflicting {name} companion archive hash identity") + + def ordinary_dependency_files(self, name, dependency): + archive = Path(dependency["installed_archive"]) + unit = dependency["original_unit"] + cmx = archive.with_suffix(".cmx") + if sha(cmx) != unit["cmx_sha256"]: + raise ValueError(f"{name} original unit cmx hash differs") + member = subprocess.check_output(["ar", "-p", str(archive.with_suffix(".a")), name + ".o"]) + if hashlib.sha256(member).hexdigest() != unit["object_sha256"]: + raise ValueError(f"{name} original unit object hash differs") + checkout = Path(dependency["source_checkout"]) + commit = dependency["pinned_commit"] + if (not checkout.is_absolute() or not (checkout / ".git").exists() + or self.run("git", "-C", checkout, "rev-parse", "HEAD") != commit + or self.run("git", "-C", checkout, "status", "--porcelain", "--untracked-files=no")): + raise ValueError(f"{name} source checkout differs from its pinned commit") + sources = dependency["installed_sources_matched_to_pinned_git"] + if not sources: + raise ValueError(f"{name} installed source identity is missing") + inputs = {str(cmx): unit["cmx_sha256"]} + for filename, record in sources.items(): + git_path = record["git_path"] + if any(Path(path).is_absolute() or ".." in Path(path).parts for path in (filename, git_path)): + raise ValueError(f"{name} source path must stay within its recorded root") + try: + data = subprocess.check_output(["git", "-C", str(checkout), "show", commit + ":" + git_path], + stderr=subprocess.PIPE) + except subprocess.CalledProcessError as error: + raise ValueError(f"{name} pinned source path is missing: {git_path}") from error + if hashlib.sha256(data).hexdigest() != record["sha256"]: + raise ValueError(f"{name} installed source hash differs from pinned Git bytes: {filename}") + inputs[str(checkout / git_path)] = record["sha256"] + return inputs + def imported_native(self, source): """Read producer records as data; never execute their recorded commands.""" envelope = json.loads(self.args.native_manifest.read_text()) - if envelope["schema_version"] != 1: + version = envelope.get("schema_version") + if type(version) is not int or version not in (1, 2): raise ValueError("unsupported native manifest schema_version") + self.native_schema_version = version directory = self.args.native_manifest.resolve().parent - pre, link, xmlm = [read_evidence(envelope[name], directory) - for name in ("source", "link", "xmlm_force_link")] + pre, link = [read_evidence(envelope[name], directory) for name in ("source", "link")] + xmlm = read_evidence(envelope["xmlm_force_link"], directory) if version == 1 else None + if version == 2 and "xmlm_force_link" in envelope: + raise ValueError("ordinary-archive schema must not reinterpret a legacy XMLm force-link proof") if pre["lui"] != self.lui_sha: raise ValueError("producer LUI SHA differs from this checkout") # Build/tool/docs commits may advance HEAD; native bytes/configuration below must still match exactly. @@ -293,13 +437,19 @@ def imported_native(self, source): expected = re.search(r'"' + name + r'\.dev"\s*"git\+[^"#]+#([0-9a-f]{40})"', lock).group(1) if link["new_dependency_provenance"][name]["pinned_commit"] != expected: raise ValueError(f"producer {name} differs from locked commit") + if version == 2: + self.ordinary_native_inputs(pre, link) if "validated_build" in envelope: inputs, historical = self.validated_native(envelope, directory, pre, link) else: - inputs = self.producer_files(pre, link, xmlm) + inputs = self.producer_files(pre, link, xmlm, version=version) historical = {} - inputs.update({str(host): link["production_host_sha256"], - str(self.args.native_manifest.resolve()): sha(self.args.native_manifest)}) + additions = {str(host): link["production_host_sha256"], + str(self.args.native_manifest.resolve()): sha(self.args.native_manifest)} + if version == 2: + self.bind_native_inputs(inputs, additions) + else: + inputs.update(additions) for record in envelope.values(): if isinstance(record, dict) and "path" in record: inputs[str((directory / record["path"]).resolve())] = record["sha256"] @@ -308,38 +458,70 @@ def imported_native(self, source): if sha(path) != digest: raise ValueError(f"native input hash mismatch: {path}") self.verify_target(host, "MACOS") + units = self.verify_ocaml_units(host, require_xmlm=version == 1) + if version == 2 and "validated_build" in envelope and units != self.receipt_ocaml_units: + raise ValueError("receipt unit retention differs from the fixed host object") symbols = self.run("nm", "-j", "-U", host) - if not all(name in symbols for name in ("_caml_startup", "_camlMldoc", "_camlXmlm")): - raise ValueError("host complete object must retain runtime, mldoc and XMLm") if any(name in symbols for name in ("_logseq_transport_dns_start", "_logseq_journal_crypto_call")): raise ValueError("host object still provides DNS/crypto; target C must provide them") self.host_object, self.native_files = host, inputs self.report["native_source_sha"] = pre["commit"] self.report["abi_limit"] = "EXPERIMENTAL: host ARM64 OCaml/runtime restamped; full iOS runtime ABI is not validated" print(self.report["abi_limit"], file=sys.stderr) - return {"producer": envelope, "files": inputs, "historical_inputs": historical, + return {"schema_version": version, "ocaml_units": units, + "producer": envelope, "files": inputs, "historical_inputs": historical, "compiler_version": link["compiler_version"], "producer_command": link["command"], "source_sha": pre["commit"], "pins": {key: pre[key] for key in ("lui", "signal", "datascript")}, - "xmlm": "single-unit force-link; original cmx/object bytes verified"} + "xmlm": ("single-unit force-link; original cmx/object bytes verified" if version == 1 else + "ordinary archive; original cmx/object bytes verified")} def validated_native(self, envelope, directory, pre, link): """Reuse a fixed complete object, with a hash-bound prior validation receipt.""" receipt = read_evidence(envelope["validated_build"], directory) previous = receipt["inputs"]["dependencies"] + version = envelope["schema_version"] if (receipt["status"] != "built" or receipt["mode"] != "experimental-host-restamp" or previous.get("source_sha", receipt["source_sha"]) != pre["commit"] - or not receipt["production_link"]["runtime_mldoc_xmlm_retained"] - or previous["compiler_version"] != link["compiler_version"] - or previous["xmlm"] != "single-unit force-link; original cmx/object bytes verified"): + or previous["compiler_version"] != link["compiler_version"]): raise ValueError("validated build receipt does not prove this producer object") - for name in ("source", "link", "xmlm_force_link"): + if version == 1: + if (not receipt["production_link"].get("runtime_mldoc_xmlm_retained") + or previous["xmlm"] != "single-unit force-link; original cmx/object bytes verified"): + raise ValueError("legacy receipt does not prove runtime/mldoc/XMLm retention") + else: + units = previous.get("ocaml_units") + if (previous.get("schema_version") != 2 or not self.valid_ocaml_units(units) + or receipt["production_link"].get("ocaml_units") != units + or "runtime_mldoc_xmlm_retained" in receipt["production_link"] + or previous.get("xmlm") != "ordinary archive; original cmx/object bytes verified"): + raise ValueError("receipt does not prove ordinary-archive unit retention") + self.receipt_ocaml_units = units + evidence_names = ("source", "link", "xmlm_force_link") if version == 1 else ("source", "link") + if previous["producer"].get("schema_version") != version: + raise ValueError("receipt producer schema differs") + for name in evidence_names: if previous["producer"][name] != envelope[name]: raise ValueError(f"validated build receipt producer evidence differs: {name}") verified = {**previous.get("historical_inputs", {}), **previous["files"]} - expected = {**link["direct_native_input_sha256"], - pre["production_native_object"]: link["production_host_sha256"], - link["command"][0]: link["compiler_sha256"]} - for name in ("source", "link", "xmlm_force_link"): + expected = {} if version == 2 else dict(link["direct_native_input_sha256"]) + def add_expected(values): + if version == 2: + self.bind_native_inputs(expected, values) + else: + expected.update(values) + if version == 2: + add_expected(link["direct_native_input_sha256"]) + add_expected({pre["production_native_object"]: link["production_host_sha256"], + link["command"][0]: link["compiler_sha256"]}) + if version == 2: + for name, dependency in link["new_dependency_provenance"].items(): + archive = Path(dependency["installed_archive"]) + add_expected({str(archive): dependency["archive_sha256"], + str(archive.with_suffix(".a")): dependency["static_archive_sha256"], + str(archive.with_suffix(".cmx")): dependency["original_unit"]["cmx_sha256"]}) + add_expected({str(archive.parent / filename): record["sha256"] + for filename, record in dependency["installed_sources_matched_to_pinned_git"].items()}) + for name in evidence_names: record = envelope[name] expected[str((directory / record["path"]).resolve())] = record["sha256"] if any(verified.get(path) != digest for path, digest in expected.items()): @@ -354,22 +536,50 @@ def validated_native(self, envelope, directory, pre, link): "consumed_inputs": "historical hash proof only; old archive paths are not live build inputs"} return headers, verified - def producer_files(self, pre, link, xmlm): + def producer_files(self, pre, link, xmlm, *, version=1): """First import verifies still-present producer inputs and XMLm bytes in full.""" - inputs = dict(link["direct_native_input_sha256"]) + inputs = {} if version == 2 else dict(link["direct_native_input_sha256"]) + def add(values): + if version == 2: + self.bind_native_inputs(inputs, values) + else: + inputs.update(values) + if version == 2: + add(link["direct_native_input_sha256"]) compiler = Path(link["command"][0]) if sha(compiler) != link["compiler_sha256"]: raise ValueError("producer compiler hash mismatch") - inputs[str(compiler)] = link["compiler_sha256"] - inputs.update(pre["installed_lui_source_sha256"]) + add({str(compiler): link["compiler_sha256"]}) + add(pre["installed_lui_source_sha256"]) for name in ("mldoc", "xmlm"): dependency = link["new_dependency_provenance"][name] archive = Path(dependency["installed_archive"]) - inputs[str(archive)] = dependency["archive_sha256"] - inputs[str(archive.with_suffix(".a"))] = dependency["static_archive_sha256"] + add({str(archive): dependency["archive_sha256"], + str(archive.with_suffix(".a")): dependency["static_archive_sha256"]}) for filename, record in dependency["installed_sources_matched_to_pinned_git"].items(): - inputs[str(archive.parent / filename)] = record["sha256"] - # Only XMLm's original unit receives force-link metadata. Never broad -linkall. + add({str(archive.parent / filename): record["sha256"]}) + if version == 2: + for name in ("mldoc", "xmlm"): + add(self.ordinary_dependency_files(name, link["new_dependency_provenance"][name])) + else: + add(self.forced_xmlm_files(compiler, link, xmlm)) + # The producer removed only host DNS/crypto from a local archive copy. + command = self.ordinary_command if version == 2 else link["command"] + effect = next(Path(p).parent for p in command if p.endswith("/logseq_sync_effect_runner_impl.cmxa")) + for name in ("logseq_sync_effect_runner_impl.cmxa", "logseq_sync_effect_runner_impl.a"): + original_path = next(p for p in inputs if Path(p).name == name) + add({str(effect / name): inputs[original_path]}) + patched = effect / "liblogseq_sync_effect_runner_impl_stubs.a" + add({str(patched): link["patched_host_stub_archive_sha256"]}) + if "platform_crypto_stubs.o" in self.run("ar", "-t", patched): + raise ValueError("producer host DNS/crypto C member was not excluded") + add({str(p): sha(p) for p in self.native_include.rglob("*.h")}) + if self.run(compiler, "-version") != link["compiler_version"]: + raise ValueError("producer OCaml compiler version differs") + return inputs + + def forced_xmlm_files(self, compiler, link, xmlm): + # Legacy schema: only XMLm's original unit receives force-link metadata. command = xmlm["command"] if len(command) != 6 or command[:4] != [str(compiler), "-a", "-linkall", "-o"]: raise ValueError("XMLm proof must describe a single-unit -a -linkall archive") @@ -386,21 +596,8 @@ def producer_files(self, pre, link, xmlm): info_tool = compiler.with_name("ocamlobjinfo") if "Force link: YES" not in self.run(info_tool, forced): raise ValueError("XMLm archive lacks force-link metadata") - inputs.update({str(cmx): xmlm["original_cmx_sha256"], str(original.parent / "xmlm.cmx"): xmlm["original_cmx_sha256"], - str(info_tool): sha(info_tool)}) - # The producer removed only host DNS/crypto from a local archive copy. - effect = next(Path(p).parent for p in link["command"] if p.endswith("/logseq_sync_effect_runner_impl.cmxa")) - for name in ("logseq_sync_effect_runner_impl.cmxa", "logseq_sync_effect_runner_impl.a"): - original_path = next(p for p in inputs if Path(p).name == name) - inputs[str(effect / name)] = inputs[original_path] - patched = effect / "liblogseq_sync_effect_runner_impl_stubs.a" - inputs[str(patched)] = link["patched_host_stub_archive_sha256"] - if "platform_crypto_stubs.o" in self.run("ar", "-t", patched): - raise ValueError("producer host DNS/crypto C member was not excluded") - inputs.update({str(p): sha(p) for p in self.native_include.rglob("*.h")}) - if self.run(compiler, "-version") != link["compiler_version"]: - raise ValueError("producer OCaml compiler version differs") - return inputs + return {str(cmx): xmlm["original_cmx_sha256"], str(original.parent / "xmlm.cmx"): xmlm["original_cmx_sha256"], + str(info_tool): sha(info_tool)} def preflight(self): a = self.args @@ -492,6 +689,7 @@ def preflight(self): def compile(self, workspace): a = self.args self.phase = "native" + native_units = None if a.link_only: source = workspace / "ocaml_stub.c" source.write_text(STUB) @@ -522,12 +720,10 @@ def compile(self, workspace): native = dune_dir / "default/app/native_embed.exe.o" self.native_gmp = dune_dir / "default/app/libgmp.a" self.report["linked_gmp"] = {"path": str(self.native_gmp), "sha256": sha(self.native_gmp)} - symbols = self.run("nm", "-U", native) - for library in ("Mldoc", "Xmlm"): - if not re.search(r"caml" + library + r"(?:__|_)", symbols): - raise RuntimeError(f"native object does not retain {library}; linker rule needs review. " - "Do not reuse a historical link argv or edit protected Dune without approval.") + native_units = self.verify_ocaml_units(native, require_xmlm=False) self.report["native_object"] = {"path": str(native), "sha256": sha(native), "mode": self.report["mode"]} + if native_units is not None: + self.report["native_object"]["ocaml_units"] = native_units self.phase = "swift" scratch = workspace / "swift" native_inputs = [native, self.target_c] if a.native_manifest else [native] @@ -569,9 +765,8 @@ def verify_target(self, product, platform=None): raise RuntimeError(f"wrong Mach-O platform, expected {platform}: {product}") def verify_production_link(self, product): - symbols = self.run("nm", "-j", "-U", product) - if not all(name in symbols for name in ("_caml_startup", "_camlMldoc", "_camlXmlm")): - raise ValueError("final product dropped OCaml runtime, mldoc or XMLm") + version = self.native_schema_version + units = self.verify_ocaml_units(product, require_xmlm=version == 1) text = self.link_map.read_text() objects = {int(m[1]): m[2].strip() for line in text.splitlines() if (m := re.match(r"^\[\s*(\d+)\]\s+(.+)$", line))} @@ -590,7 +785,9 @@ def verify_production_link(self, product): raise ValueError("product links a non-system dynamic library") self.report["production_link"] = {"link_map": str(self.link_map), "link_map_sha256": sha(self.link_map), "target_c": {"path": str(self.target_c), "sha256": sha(self.target_c)}, "swift_objects": swift_objects, - "runtime_mldoc_xmlm_retained": True, "cross_abi_validated": False} + "ocaml_units": units, "cross_abi_validated": False} + if version == 1: + self.report["production_link"]["runtime_mldoc_xmlm_retained"] = True def device_signing(self): a = self.args diff --git a/.agents/skills/journal-build/scripts/test_journal_build.py b/.agents/skills/journal-build/scripts/test_journal_build.py index 8fb16470..38e7878e 100644 --- a/.agents/skills/journal-build/scripts/test_journal_build.py +++ b/.agents/skills/journal-build/scripts/test_journal_build.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Small build-boundary checks; no device, dependency install or large build.""" import importlib.util +import hashlib from contextlib import contextmanager from functools import partial import json @@ -11,6 +12,7 @@ import subprocess import tempfile import signal +import shutil import sys import threading import time @@ -784,6 +786,301 @@ def test_consumed_archives_may_change_or_disappear_with_fixed_object_receipt(sel f.history.unlink() f.build.imported_native(f.source) + @contextmanager + def ordinary_producer_fixture(self, *, entry=None, relative=False, warnings=False): + """Actual OCaml link: unused Xmlm archive supplied, unit legally omitted.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + repo, package, effect = root / 'repo', root / 'package', root / 'effect' + (repo / 'app').mkdir(parents=True) + platform = repo / 'logseq_sync/lib/effect_runner/platform' + platform.mkdir(parents=True) + for name in ('platform_crypto_stubs.c', 'platform_crypto_stubs.h'): + (platform / name).write_text('/* target source */\n') + compiler = Path(shutil.which('ocamlopt')).resolve() + runtime = Path(subprocess.check_output([compiler, '-where'], text=True).strip()) + effect.mkdir() + (effect / 'effect.ml').write_text('let value = 1\n') + def run(*argv, cwd=root): + subprocess.run(list(map(str, argv)), cwd=cwd, check=True, capture_output=True) + run(compiler, '-c', effect / 'effect.ml') + effect_archive = effect / 'logseq_sync_effect_runner_impl.cmxa' + run(compiler, '-a', '-o', effect_archive, effect / 'effect.cmx') + (effect / 'stub.c').write_text('int harmless_stub;\n') + run('xcrun', 'clang', '-c', effect / 'stub.c', '-o', effect / 'stub.o') + stub = effect / 'liblogseq_sync_effect_runner_impl_stubs.a' + run('ar', 'rcs', stub, effect / 'stub.o') + provenance = {} + for name, source in [('mldoc', 'let parse text = String.length text\n'), + ('xmlm', 'let make_input text = text\n')]: + checkout = root / (name + '-source') + checkout.mkdir() + (checkout / (name + '.ml')).write_text(source) + run('git', 'init', '-q', checkout) + run('git', '-C', checkout, 'add', '.') + run('git', '-C', checkout, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', name) + pinned = subprocess.check_output(['git', '-C', str(checkout), 'rev-parse', 'HEAD'], text=True).strip() + lib = package / 'lib' / name + lib.mkdir(parents=True) + (lib / (name + '.ml')).write_text(source) + run(compiler, '-c', lib / (name + '.ml')) + archive = lib / (name + '.cmxa') + run(compiler, '-a', '-o', archive, lib / (name + '.cmx')) + object_bytes = subprocess.check_output(['ar', '-p', str(archive.with_suffix('.a')), name + '.o']) + provenance[name] = dict(pinned_commit=pinned, source_checkout=str(checkout), + installed_archive=str(archive), archive_sha256=self.build.sha(archive), + static_archive_sha256=self.build.sha(archive.with_suffix('.a')), + installed_sources_matched_to_pinned_git={name + '.ml': dict( + sha256=self.build.sha(lib / (name + '.ml')), git_path=name + '.ml')}, + original_unit=dict(cmx_sha256=self.build.sha(lib / (name + '.cmx')), + object_sha256=hashlib.sha256(object_bytes).hexdigest())) + (repo / 'app/unit.ml').write_text(entry or 'let () = ignore (Sys.opaque_identity Mldoc.parse)\n') + run(compiler, '-I', package / 'lib/mldoc', '-c', repo / 'app/unit.ml') + native = root / 'host.o' + command = list(map(str, [compiler, '-output-complete-obj', '-o', native, effect_archive, + package / 'lib/xmlm/xmlm.cmxa', package / 'lib/mldoc/mldoc.cmxa', repo / 'app/unit.cmx'])) + if relative: + command = [command[0]] + [os.path.relpath(arg, root) if arg.startswith(str(root) + '/') else arg + for arg in command[1:]] + if warnings: + command[1:1] = ['-w', '@1..3'] + run(*command) + pins = dict(lui='1' * 40, signal='2' * 40, datascript='3' * 40) + lock_names = dict(signal='ocaml-signal', datascript='datascript-ocaml-native') + all_pins = {**pins, **{name: item['pinned_commit'] for name, item in provenance.items()}} + (repo / 'logseq_journal.opam.locked').write_text('\n'.join( + f'"{lock_names.get(name, name)}.dev" "git+https://example.invalid/{name}#{commit}"' + for name, commit in all_pins.items())) + run('git', 'init', '-q', repo) + run('git', '-C', repo, 'add', 'app/unit.ml', 'logseq_journal.opam.locked', 'logseq_sync') + run('git', '-C', repo, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', 'producer') + commit = subprocess.check_output(['git', '-C', str(repo), 'rev-parse', 'HEAD'], text=True).strip() + with patch.object(self.build, 'ROOT', repo): + source = self.build.source_inputs() + pre = dict(commit=commit, **pins, production_native_object=str(native), + production_native_object_sha256=self.build.sha(native), + native_source_sha256={'app/unit.ml': self.build.sha(repo / 'app/unit.ml')}, + installed_lui_source_sha256={}) + inputs = [effect_archive, effect_archive.with_suffix('.a'), stub, + repo / 'app/unit.cmx', repo / 'app/unit.o'] + for item in provenance.values(): + archive = Path(item['installed_archive']) + inputs.extend([archive, archive.with_suffix('.a')]) + link = dict(command=command, working_directory=str(root), compiler_sha256=self.build.sha(compiler), + compiler_version=subprocess.check_output([compiler, '-version'], text=True).strip(), + production_host_sha256=self.build.sha(native), + direct_native_input_sha256={str(p): self.build.sha(p) for p in inputs}, + device_c_source_sha256={p.name: self.build.sha(p) for p in platform.iterdir()}, + patched_host_stub_archive_sha256=self.build.sha(stub), new_dependency_provenance=provenance) + envelope = dict(schema_version=2) + path = root / 'input.json' + def save(): + for name, value in [('source', pre), ('link', link)]: + record = root / (name + '.json') + record.write_text(json.dumps(value)) + envelope[name] = dict(path=str(record), sha256=self.build.sha(record)) + path.write_text(json.dumps(envelope)) + save() + build = self.build.Build(SimpleNamespace(native_manifest=path, experimental_host_restamp=True, + link_only=False, platform='ios-simulator'), root) + build.run = partial(build.run, cwd=repo) + build.report['source_sha'], build.lui_sha = commit, pins['lui'] + yield SimpleNamespace(build=build, source=source, link=link, pre=pre, native=native, + envelope=envelope, path=path, root=root, save=save, runtime=runtime, provenance=provenance) + + def test_ordinary_xmlm_input_allows_dead_code_elimination(self): + with self.ordinary_producer_fixture() as f: + imported = f.build.imported_native(f.source) + self.assertEqual(imported['ocaml_units'], dict(runtime=True, mldoc=True, xmlm=False)) + self.assertEqual(imported['schema_version'], 2) + archive = f.provenance['xmlm']['installed_archive'] + self.assertEqual(imported['files'][archive], f.provenance['xmlm']['archive_sha256']) + + def test_ordinary_producer_accepts_real_warning_and_relative_argv(self): + for option in ('warnings', 'relative'): + with self.subTest(option=option), self.ordinary_producer_fixture(**{option: True}) as f: + self.assertEqual(f.build.imported_native(f.source)['ocaml_units']['xmlm'], False) + + def test_ordinary_producer_rejects_unbound_source_and_linker_arguments(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) + original = list(f.link['command']) + for args in ([str(f.root / 'unbound.ml')], [str(f.root / 'unbound.c')], + ['-cclib', '-lunbound'], ['-ccopt', str(f.root / 'unbound.o')]): + with self.subTest(args=args): + f.link['command'] = original + args + f.save() + with self.assertRaisesRegex(ValueError, 'input|argument|unsupported|library'): + f.build.imported_native(f.source) + + def test_ordinary_producer_rejects_conflicting_companion_identity(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) + archive = Path(f.provenance['xmlm']['installed_archive']).with_suffix('.a') + f.link['direct_native_input_sha256'][str(archive)] = 'f' * 64 + f.save() + with self.assertRaisesRegex(ValueError, 'conflict|hash|identity'): + f.build.imported_native(f.source) + + def test_ordinary_producer_rejects_split_cclib_input(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) + extra = f.root / 'unrecorded.o' + shutil.copyfile(f.root / 'effect/stub.o', extra) + f.link['command'].extend(['-cclib', '-L' + str(f.root) + ' ' + str(extra)]) + subprocess.run(f.link['command'], cwd=f.root, check=True, capture_output=True) + f.pre['production_native_object_sha256'] = self.build.sha(f.native) + f.link['production_host_sha256'] = self.build.sha(f.native) + f.save() + with self.assertRaisesRegex(ValueError, 'input|argument|token'): + f.build.imported_native(f.source) + + def test_ordinary_producer_rejects_cclib_option_disguised_as_object(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) + extra = f.root / 'extra.o' + shutil.copyfile(f.root / 'effect/stub.o', extra) + decoy = f.root / '-lextra.o' + decoy.write_bytes(b'not the object selected by Darwin ld\n') + f.link['direct_native_input_sha256'][str(decoy)] = self.build.sha(decoy) + f.link['command'].extend(['-cclib', '-L' + str(f.root), '-cclib', '-lextra.o']) + subprocess.run(f.link['command'], cwd=f.root, check=True, capture_output=True) + f.pre['production_native_object_sha256'] = self.build.sha(f.native) + f.link['production_host_sha256'] = self.build.sha(f.native) + f.save() + with self.assertRaisesRegex(ValueError, 'input|argument|unsupported'): + f.build.imported_native(f.source) + + def test_ordinary_receipt_survives_removed_historical_cmx_symlink(self): + with self.ordinary_producer_fixture() as f: + cmx = Path(f.provenance['xmlm']['installed_archive']).with_suffix('.cmx') + saved = f.root / 'historical-xmlm.cmx' + cmx.rename(saved) + cmx.symlink_to(saved) + imported = f.build.imported_native(f.source) + receipt = dict(status='built', mode='experimental-host-restamp', source_sha=f.pre['commit'], + production_link=dict(ocaml_units=imported['ocaml_units'], cross_abi_validated=False), + inputs=dict(dependencies=imported)) + path = f.root / 'receipt.json' + path.write_text(json.dumps(receipt)) + f.envelope['validated_build'] = dict(path=str(path), sha256=self.build.sha(path)) + f.path.write_text(json.dumps(f.envelope)) + f.build.imported_native(f.source) + cmx.unlink() + imported = f.build.imported_native(f.source) + self.assertNotIn(str(cmx), imported['files']) + self.assertIn(str(cmx), imported['historical_inputs']) + + def test_ordinary_producer_rejects_changed_dependency_and_actual_inputs(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) # Prove the valid baseline first. + original = json.loads(json.dumps(f.link)) + cases = ('pin', 'compiler', 'archive', 'static_archive', 'cmx', 'object', 'source', + 'source_git_path', 'linked_archive', 'unrecorded_input', 'extra_input') + for case in cases: + with self.subTest(case=case): + f.link.clear(); f.link.update(json.loads(json.dumps(original))) + item = f.link['new_dependency_provenance']['xmlm'] + if case == 'pin': item['pinned_commit'] = 'f' * 40 + elif case == 'compiler': f.link['compiler_sha256'] = 'f' * 64 + elif case == 'archive': item['archive_sha256'] = 'f' * 64 + elif case == 'static_archive': item['static_archive_sha256'] = 'f' * 64 + elif case in ('cmx', 'object'): item['original_unit'][case + '_sha256'] = 'f' * 64 + elif case == 'source': item['installed_sources_matched_to_pinned_git']['xmlm.ml']['sha256'] = 'f' * 64 + elif case == 'source_git_path': item['installed_sources_matched_to_pinned_git']['xmlm.ml']['git_path'] = 'missing.ml' + elif case == 'linked_archive': + f.link['command'].remove(item['installed_archive']) + elif case == 'unrecorded_input': + f.link['direct_native_input_sha256'].pop(item['installed_archive']) + else: + other = f.root / 'unrecorded.cmxa' + shutil.copyfile(item['installed_archive'], other) + f.link['command'].append(str(other)) + f.save() + with self.assertRaisesRegex((ValueError, RuntimeError), 'commit|compiler|hash|XMLm|input|archive|unit|source|path'): + f.build.imported_native(f.source) + + def test_ordinary_producer_still_requires_used_mldoc(self): + with self.ordinary_producer_fixture(entry='let () = ()\n') as f: + with self.assertRaisesRegex(ValueError, 'mldoc|Mldoc'): + f.build.imported_native(f.source) + + def test_ordinary_producer_rejects_missing_runtime_and_unresolved_xmlm(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) + for case in ('runtime', 'unresolved Xmlm'): + with self.subTest(case=case): + c = f.root / 'invalid.c' + c.write_text('int camlMldoc;\n' + ('' if case == 'runtime' else + 'void caml_startup(void) {}\nextern int camlXmlm; int *reference = &camlXmlm;\n')) + subprocess.run(['xcrun', 'clang', '-c', c, '-o', f.native], check=True, capture_output=True) + f.pre['production_native_object_sha256'] = self.build.sha(f.native) + f.link['production_host_sha256'] = self.build.sha(f.native) + f.save() + with self.assertRaisesRegex(ValueError, 'runtime|unresolved|XMLm|Xmlm'): + f.build.imported_native(f.source) + + def test_ordinary_final_link_reports_xmlm_absence_without_all_retained_claim(self): + with self.ordinary_producer_fixture() as f: + f.build.imported_native(f.source) + f.build.target_c = f.root / 'target.o' + c = f.root / 'target.c' + c.write_text('void logseq_transport_dns_start(void) {}\n' + 'void logseq_journal_crypto_call(void) {}\nint main(void) { return 0; }\n') + subprocess.run(['xcrun', 'clang', '-c', c, '-o', f.build.target_c], check=True, capture_output=True) + product = f.root / 'JournalApp' + f.build.link_map = f.root / 'link-map.txt' + subprocess.run(['xcrun', 'clang', f.native, f.build.target_c, '-o', product, + '-Wl,-map,' + str(f.build.link_map)], check=True, capture_output=True) + objects = f.root / 'JournalApp.product' + objects.mkdir() + (objects / 'Objects.LinkFileList').write_text('') + f.build.verify_production_link(product) + observed = f.build.report['production_link'] + self.assertEqual(observed['ocaml_units'], dict(runtime=True, mldoc=True, xmlm=False)) + self.assertNotIn('runtime_mldoc_xmlm_retained', observed) + self.assertIs(observed['cross_abi_validated'], False) + + def test_ordinary_receipt_records_actual_retention_and_keeps_history(self): + with self.ordinary_producer_fixture() as f: + imported = f.build.imported_native(f.source) + expected_units = dict(imported['ocaml_units']) + receipt = dict(status='built', mode='experimental-host-restamp', source_sha=f.pre['commit'], + production_link=dict(ocaml_units=imported['ocaml_units'], cross_abi_validated=False), + inputs=dict(dependencies=imported)) + receipt_path = f.root / 'receipt.json' + def save_receipt(): + receipt_path.write_text(json.dumps(receipt)) + f.envelope['validated_build'] = dict(path=str(receipt_path), sha256=self.build.sha(receipt_path)) + f.path.write_text(json.dumps(f.envelope)) + save_receipt() + accepted = f.build.imported_native(f.source) + self.assertEqual(accepted['ocaml_units']['xmlm'], False) + receipt['production_link']['ocaml_units'] = dict(runtime=True, mldoc=True, xmlm=True) + save_receipt() + with self.assertRaisesRegex(ValueError, 'receipt|retention|unit'): + f.build.imported_native(f.source) + receipt['inputs']['dependencies']['ocaml_units'] = dict(runtime=True, mldoc=True, xmlm=True) + save_receipt() + with self.assertRaisesRegex(ValueError, 'receipt|retention|unit'): + f.build.imported_native(f.source) + receipt['production_link'].pop('ocaml_units') + receipt['production_link']['runtime_mldoc_xmlm_retained'] = True + save_receipt() + with self.assertRaisesRegex(ValueError, 'receipt|retention|unit'): + f.build.imported_native(f.source) + receipt['production_link'].pop('runtime_mldoc_xmlm_retained') + receipt['production_link']['ocaml_units'] = dict(expected_units) + receipt['inputs']['dependencies']['ocaml_units'] = dict(expected_units) + save_receipt() + archive = Path(f.provenance['xmlm']['installed_archive']) + archive.unlink() + accepted = f.build.imported_native(f.source) + self.assertIn(str(archive), accepted['historical_inputs']) + self.assertNotIn(str(archive), accepted['files']) + def test_builder_only_commit_keeps_exact_native_source_identity(self): with self.producer_fixture() as f: (f.repo / 'tool').mkdir() diff --git a/docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md b/docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md new file mode 100644 index 00000000..48cf8a24 --- /dev/null +++ b/docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md @@ -0,0 +1,39 @@ +# Allow Unused Xmlm Native Input + +## Problem + +The native builder rejects a complete object whose locked XMLm archive is a correct link input but whose unused XMLm unit was removed by the OCaml linker. Journal's Markdown title path calls mldoc without using its OPML parser/exporter. The current unconditional XMLm symbol requirement confuses input identity with runtime reachability. + +## Proposal + +The user explicitly approved build-tool and provenance-contract corrections on 2026-10-17. Keep dependency versions, source pins, compiler, actual native link inputs and their byte identities strict. Allow XMLm to be absent only when no XMLm reference remains unresolved. Record observed runtime/mldoc/XMLm retention separately from input identity. Runtime and mldoc remain required. + +Preserve schema 1 force-link producers and their immutable receipts unchanged. Add schema 2 for an ordinary original XMLm archive, binding its original cmx/object bytes and installed source identities as well as the complete object's actual compiler/link inputs. A new receipt records the observed unit-retention map accurately; it cannot reuse the old all-retained boolean for an object without XMLm. Recheck live object/headers/evidence as before, with receipt-bound historical archives remaining historical. + +Change only the existing build owner, its narrow boundary tests and guidance. Do not change Dune, native_embed, dependency pins, protected interfaces, phones or graphs. Coordinate compilation and GUI with the Timeline owner before executing tests/builds. + +## Alternatives considered + +### Retain XMLm with an application anchor + +Rejected: this adds unused production reachability to satisfy a check without a product runtime requirement. The previously drafted Dune/anchor patch remains unapplied. + +### Delete only the macOS symbol check + +Rejected: experimental import, final product checks and receipt validation would still enforce the old force-link contract, while manifest fields could misreport actual retention. + +## Acceptance criteria + +- A real tiny ordinary-archive producer with runtime/mldoc and no XMLm symbols is accepted with accurate retention observations and strict input hashes. +- Wrong dependency pins, changed compiler/source/archive/cmx/object/link-input identity, unresolved XMLm references and missing required runtime/mldoc symbols are rejected. +- Legacy producer/receipt regressions remain valid, including evidence tampering, changed native source and runtime-header checks. A new receipt cannot falsify observed unit retention or reinterpret a legacy receipt. +- Run RED before production edits, then the build-owner suite. Independently review the provenance contract before accepting new product artifacts; perform fresh final source/build/tests and A135 read-only acceptance afterward. + +## Risks + +- A permissive receipt/schema transition could accept unsupported inputs; versioned records, exact actual-input membership and negative tests must keep the change confined to legal dead-code elimination. +- Host-restamped iOS remains experimental with cross_abi_validated=false. + +## Questions + +- None. The user approved this exact direction; no protected Dune/spec edit is needed. From 994cfd5e609c2aa5398b8cc59179fcb4ca4cde57 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 17 Oct 2026 14:23:01 +0800 Subject: [PATCH 3/5] Record verified hydration and native provenance decisions --- ...26-10-17-unify-worker-request-scheduling.md | 18 ++++++++++++++++++ ...026-10-17-allow-unused-xmlm-native-input.md | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) rename docs/agent-guide/{proposed => implemented}/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md (54%) rename docs/agent-guide/{proposed => implemented}/bugfix/2026-10-17-allow-unused-xmlm-native-input.md (57%) diff --git a/docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md b/docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md similarity index 54% rename from docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md rename to docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md index 4e78fbac..3865d001 100644 --- a/docs/agent-guide/proposed/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md +++ b/docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md @@ -8,6 +8,10 @@ A Favorite block completion drains hydration internally and the public receive b The user approved unified App-to-DB-Worker scheduling on 2026-10-17. Internal runtime transitions produce demands and mark their priority without dispatching. Public runtime entry points alone enqueue and drain once; queued and active identity sets are disjoint and deduplicated. Retain the shared four-read background bound and existing foreground admission. Transport delivers each output once and retires rejected unsent demands explicitly without retrying Worker.send. Application shares graph request admission and ownership routing instead of separate Favorites/admission/general delivery loops; Copy/media/assets retain their own state machines. Include Page titles in the existing reference observer and use the same title renderer. +## Decision + +Use private Planned/Queued/Dispatched delivery state in the existing graph runtime, one public dispatch boundary and one Application Worker adapter. Route admission, Favorites and ordinary graph outputs through the same transport and typed completion path. Complete the existing Page title observer and shared renderer wiring without changing the public protocol or protected interfaces. + ## Alternatives considered ### Increase the background or Worker limit @@ -18,6 +22,12 @@ Rejected: it masks lost delivery and cannot bound arbitrary graph fan-out. Rejected: one graph-session owner already controls reference epochs and cache. Rejection must settle ownership explicitly, not silently resubmit work. +## Consequences + +Every background slot now corresponds to a request returned for delivery. Refusal retires unsent ownership, including work released by failure completions, without another send attempt. Copy, media and asset state machines keep their domain ownership while sharing physical Worker admission. Graph scope cancellation and duplicate protocol-ID protection live in the adapter. The background limit remains four and Worker concurrency remains two. + +Page-first lookup still falls back to an outline block only after an explicit missing Page result. UUID references expand to ordinary non-clickable text. No deprecated double-parenthesis reference syntax, new resolver, Dune change, dependency pin change, graph edit or phone operation is included. + ## Acceptance criteria - Public Runtime regressions drain at least four Favorite block dependencies and queued references exactly once; failures, cancellation, duplicate completions, reset and epoch changes preserve capacity and fencing. @@ -37,3 +47,11 @@ Rejected: one graph-session owner already controls reference epochs and cache. R ## Questions - None. The user explicitly approved this scope; no protected interface change is currently proposed. + +## Verification + +New public Runtime and narrow mounted Application owner regressions failed before the production changes and pass afterward. A fresh private locked switch passed the full Release build and tests: 21 Alcotest suites with 906 cases, plus 87 standalone PASS records. A complete Application/Worker replay of a consistent real-data copy sent and received the target once and resolved both Favorites and detail titles. + +A135 was updated in place to the formal Simulator build 1.5877.34, source/native revision 6ab0ead. A cold-start test explicitly selected the existing Lambda RTC graph, opened Favorites and the original target detail, and returned normally: both title hashes matched the resolved result, rawToken=false, links=0, and the detail was static text. The single UI case passed in 17.992 seconds; graph outbox counts were zero before and after. No credentials were entered and no graph edit was performed. Favorite Page-specific cases are covered at the public owner boundary, not claimed as a separate real-graph Page example. + +The Simulator uses experimental Host ARM64 restamping with cross_abi_validated=false; it does not establish a fully cross-compiled OCaml iOS ABI. Formal PR review and exact-head CI are the remaining publication gates. Two new documents validate individually; the all-document check retains three previously recorded unrelated format failures. diff --git a/docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md b/docs/agent-guide/implemented/bugfix/2026-10-17-allow-unused-xmlm-native-input.md similarity index 57% rename from docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md rename to docs/agent-guide/implemented/bugfix/2026-10-17-allow-unused-xmlm-native-input.md index 48cf8a24..a9ad2f69 100644 --- a/docs/agent-guide/proposed/bugfix/2026-10-17-allow-unused-xmlm-native-input.md +++ b/docs/agent-guide/implemented/bugfix/2026-10-17-allow-unused-xmlm-native-input.md @@ -12,6 +12,10 @@ Preserve schema 1 force-link producers and their immutable receipts unchanged. A Change only the existing build owner, its narrow boundary tests and guidance. Do not change Dune, native_embed, dependency pins, protected interfaces, phones or graphs. Coordinate compilation and GUI with the Timeline owner before executing tests/builds. +## Decision + +Add schema 2 for ordinary original archives and keep schema 1 unchanged. Bind expanded object-only argv, compiler, original archive companions, unit/object bytes and pinned Git source identities. Require runtime/mldoc retention and reject unresolved XMLm symbols, while permitting unused XMLm elimination. Record the actual three-unit boolean map in schema-2 imports and final links instead of the legacy all-retained claim. + ## Alternatives considered ### Retain XMLm with an application anchor @@ -22,6 +26,12 @@ Rejected: this adds unused production reachability to satisfy a check without a Rejected: experimental import, final product checks and receipt validation would still enforce the old force-link contract, while manifest fields could misreport actual retention. +## Consequences + +Unknown compiler/linker inputs, split or disguised -cclib options, response files, conflicting input hashes and output-as-input identities are refused. Relative argv uses its recorded working directory. Absolute historical input names are stable lexical identities, so removing an installation symlink after a valid receipt does not change the identity being checked. Current source, object, runtime headers and evidence remain live checks; only receipt-bound consumed inputs can become historical. + +No production anchor, forced archive, Dune/spec edit or shared dependency repair is needed. Experimental Host restamping remains explicitly outside full iOS ABI validation. + ## Acceptance criteria - A real tiny ordinary-archive producer with runtime/mldoc and no XMLm symbols is accepted with accurate retention observations and strict input hashes. @@ -37,3 +47,11 @@ Rejected: experimental import, final product checks and receipt validation would ## Questions - None. The user approved this exact direction; no protected Dune/spec edit is needed. + +## Verification + +Real tiny compiler/linker fixtures first reproduced the new contract and seven independent review findings as RED cases. After the corrections all 46 build-owner tests pass with no skips, including legacy producer/receipt cases and the installed SQLite runtime boundary. A separate read-only journal-code-review session confirmed the seven findings closed and no further evidence-backed contract issues; this does not replace the later whole-PR review. + +The formal native macOS phase for revision 6ab0ead succeeded with runtime=true, mldoc=true and xmlm=false. The macOS App subsequently failed on the pre-existing JournalChrome listSectionSpacing API being unavailable on macOS, so no macOS App success is claimed. Fresh source/link records use that completed native phase's actual Dune argv and current input hashes; the old failed overall manifest is not a successful receipt. + +The formal Simulator build 1.5877.34 completed input revalidation, target C/Swift compilation, final symbol/provider checks and signing with accurate schema-2 unit observations. A135 read-only Lambda RTC Favorites/detail acceptance then passed. The artifact remains experimental-host-restamp with cross_abi_validated=false. From 63fce61a79179b8dc3e3f69ee0520233c8626b7a Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 17 Oct 2026 14:29:20 +0800 Subject: [PATCH 4/5] Use the prepared switch for native provenance fixtures --- .../journal-build/scripts/test_journal_build.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.agents/skills/journal-build/scripts/test_journal_build.py b/.agents/skills/journal-build/scripts/test_journal_build.py index 38e7878e..b2e91dc1 100644 --- a/.agents/skills/journal-build/scripts/test_journal_build.py +++ b/.agents/skills/journal-build/scripts/test_journal_build.py @@ -797,7 +797,17 @@ def ordinary_producer_fixture(self, *, entry=None, relative=False, warnings=Fals platform.mkdir(parents=True) for name in ('platform_crypto_stubs.c', 'platform_crypto_stubs.h'): (platform / name).write_text('/* target source */\n') - compiler = Path(shutil.which('ocamlopt')).resolve() + switch = os.environ.get('JOURNAL_TEST_SWITCH') + if switch: + bindir = subprocess.check_output( + ['opam', 'var', '--safe', '--switch', switch, 'bin'], text=True).strip() + compiler = Path(bindir) / 'ocamlopt.opt' + else: + selected = shutil.which('ocamlopt') + if selected is None: + self.fail('set JOURNAL_TEST_SWITCH to a prepared switch or expose ocamlopt on PATH') + compiler = Path(selected) + compiler = compiler.resolve() runtime = Path(subprocess.check_output([compiler, '-where'], text=True).strip()) effect.mkdir() (effect / 'effect.ml').write_text('let value = 1\n') From fec63d8f97a0dbf5c3cf519f59bcc6a45cbf1978 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 17 Oct 2026 15:05:54 +0800 Subject: [PATCH 5/5] Retain retired graph Worker leases through terminal delivery --- app/application.ml | 39 ++++++++++---- ...6-10-17-unify-worker-request-scheduling.md | 5 ++ test/application_view_test.ml | 51 ++++++++++++++++++- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/app/application.ml b/app/application.ml index 3cbfac2c..9148c4b6 100644 --- a/app/application.ml +++ b/app/application.ml @@ -3678,6 +3678,11 @@ let response_tag = function | tag -> tag ;; +type graph_worker_lease = + { protocol_request : Logseq_db_worker.Protocol.request + ; mutable retired : bool + } + let start ~on_view_region ~on_platform_request @@ -3763,23 +3768,29 @@ let start match Worker.send client request with | Accepted worker_id as accepted -> Option.iter (fun protocol -> - Hashtbl.add admitted_graph_requests worker_id protocol; + Hashtbl.add admitted_graph_requests worker_id + { protocol_request = protocol; retired = false }; Hashtbl.add admitted_protocol_ids protocol.Logseq_db_worker.Protocol.request_id worker_id; if observe_projection then !projection_request_accepted protocol) protocol; accepted | (Full | Not_ready | Stopping) as refused -> refused in let retire_worker_lease worker_id = - Option.iter (fun request -> - Hashtbl.remove admitted_protocol_ids request.Logseq_db_worker.Protocol.request_id) - (Hashtbl.find_opt admitted_graph_requests worker_id); - Hashtbl.remove admitted_graph_requests worker_id + match Hashtbl.find_opt admitted_graph_requests worker_id with + | None -> false + | Some lease -> + let protocol_id = lease.protocol_request.request_id in + if Hashtbl.find_opt admitted_protocol_ids protocol_id = Some worker_id then + Hashtbl.remove admitted_protocol_ids protocol_id; + Hashtbl.remove admitted_graph_requests worker_id; + lease.retired in let cancel_graph_leases () = - Hashtbl.iter (fun worker_id request -> - !projection_request_finished request.Logseq_db_worker.Protocol.request_id; - Worker.cancel client ~request_id:worker_id) admitted_graph_requests; - Hashtbl.clear admitted_graph_requests; + Hashtbl.iter (fun worker_id lease -> + if not lease.retired then ( + lease.retired <- true; + !projection_request_finished lease.protocol_request.request_id; + Worker.cancel client ~request_id:worker_id)) admitted_graph_requests; Hashtbl.clear admitted_protocol_ids in let cancel_copy () = @@ -4402,8 +4413,9 @@ let start ]) in let handle_worker_event event = - (match event with Worker.Response { request_id; _ } -> retire_worker_lease request_id - | _ -> ()); + let retired = match event with + | Worker.Response { request_id; _ } -> retire_worker_lease request_id + | _ -> false in Journal_asset_runtime.pump asset_runtime; sync_media !state_ref; Journal_media_runtime.pump media_runtime; @@ -4541,6 +4553,9 @@ let start forget_projection_request protocol_id; Journal_asset_runtime.reject asset_runtime ~request_id:protocol_id); Effect.ignore + | Worker.Response { outcome = Completed (Graph_service.Graph_response _); _ } + when retired -> + Effect.ignore | Worker.Response { request_id ; outcome = Worker.Completed (Graph_service.Graph_response response) @@ -4813,6 +4828,8 @@ let start | None -> state | Some message when Option.is_some state.feed_refresh -> fail_feed_transport state message | Some message -> fail_graph_transport state message))) + | Worker.Response { outcome = Failed _ | Cancelled | Shutdown; _ } when retired -> + Effect.ignore | Worker.Response { outcome = Failed error; _ } -> set_state (fun state -> let worker_error = service_error ~operation:"handleRequest" error in diff --git a/docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md b/docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md index 3865d001..a3b21600 100644 --- a/docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md +++ b/docs/agent-guide/implemented/bugfix/2026-10-17-2026-10-17-unify-worker-request-scheduling.md @@ -26,6 +26,8 @@ Rejected: one graph-session owner already controls reference epochs and cache. R Every background slot now corresponds to a request returned for delivery. Refusal retires unsent ownership, including work released by failure completions, without another send attempt. Copy, media and asset state machines keep their domain ownership while sharing physical Worker admission. Graph scope cancellation and duplicate protocol-ID protection live in the adapter. The background limit remains four and Worker concurrency remains two. +Cancelled graph scopes retain retired Worker-ID leases until the guaranteed terminal response. These leases cannot remove a newer protocol-ID admission or publish a current-scope error. Domain-specific Copy, media and asset response handlers still perform their own cleanup before the generic retired-response guard. + Page-first lookup still falls back to an outline block only after an explicit missing Page result. UUID references expand to ordinary non-clickable text. No deprecated double-parenthesis reference syntax, new resolver, Dune change, dependency pin change, graph edit or phone operation is included. ## Acceptance criteria @@ -33,6 +35,7 @@ Page-first lookup still falls back to an outline block only after an explicit mi - Public Runtime regressions drain at least four Favorite block dependencies and queued references exactly once; failures, cancellation, duplicate completions, reset and epoch changes preserve capacity and fencing. - Foreground reads remain admitted while four background demands are outstanding; only explicit missing Page results trigger outline fallback. - Transport refusal releases all unsent ownership and reports failure without hidden send retries. +- An actual delayed old-scope Worker cancellation cannot publish a sync error after selecting another graph. - Favorite Page titles and detail Page roots discover references; display uses shared lookup and creates no reference link. - Existing targeted tests and full build/tests pass; validate controlled fixtures and the existing A135 Lambda RTC read-only flow with identified artifacts. - Do not modify Dune, protected spec interfaces, real graph data or the phone. Any required protected interface change needs a precise separately approved diff. @@ -52,6 +55,8 @@ Page-first lookup still falls back to an outline block only after an explicit mi New public Runtime and narrow mounted Application owner regressions failed before the production changes and pass afterward. A fresh private locked switch passed the full Release build and tests: 21 Alcotest suites with 906 cases, plus 87 standalone PASS records. A complete Application/Worker replay of a consistent real-data copy sent and received the target once and resolved both Favorites and detail titles. +Independent PR review found a scope-retirement gap in the App adapter: clearing the accepted Worker-ID map let a delayed cancellation fall into the current-scope generic error path. The pure graph Runtime has no Worker IDs or App sync-error state, so one regression uses the existing mounted Application public hooks with the real Worker. It holds cancellable cleanup of an old Favorite read, selects a new graph, and releases the actual terminal response. The unchanged implementation failed with a visible "Worker unavailable" message; retaining the retired lease passes. This adds one Application owner case (120 total); the full final build/test and new native artifact verification are recorded in the PR evidence. + A135 was updated in place to the formal Simulator build 1.5877.34, source/native revision 6ab0ead. A cold-start test explicitly selected the existing Lambda RTC graph, opened Favorites and the original target detail, and returned normally: both title hashes matched the resolved result, rawToken=false, links=0, and the detail was static text. The single UI case passed in 17.992 seconds; graph outbox counts were zero before and after. No credentials were entered and no graph edit was performed. Favorite Page-specific cases are covered at the public owner boundary, not claimed as a separate real-graph Page example. The Simulator uses experimental Host ARM64 restamping with cross_abi_validated=false; it does not establish a fully cross-compiled OCaml iOS ABI. Formal PR review and exact-head CI are the remaining publication gates. Two new documents validate individually; the all-document check retains three previously recorded unrelated format failures. diff --git a/test/application_view_test.ml b/test/application_view_test.ml index 105ef608..f74f0c65 100644 --- a/test/application_view_test.ml +++ b/test/application_view_test.ml @@ -5363,11 +5363,16 @@ let test_graph_picker_controls scenario () = let release_automatic_inspection = Atomic.make false in let automatic_inspection_returned = Atomic.make false in let inspection_terminals = ref 0 in + let graph_read_started = Atomic.make false and graph_read_finished = Atomic.make false in + let release_graph_read = Atomic.make false and graph_read_cancellations = ref 0 in + let graph_read_cancelling = Atomic.make false and release_graph_terminal = Atomic.make false in + let graph_read_worker = Atomic.make None in let graph_opens = Atomic.make 0 in let root_renders = ref 0 in let client = ref None in let service = W.Service.create ~push_topic_count:6 - ~concurrency:(if scenario = `Late_presence then Concurrent { max_in_flight = 2 } else Serial) + ~concurrency:(if scenario = `Late_presence || scenario = `Late_graph_cancel + then Concurrent { max_in_flight = 2 } else Serial) ~init:(fun context _ -> W.Session_context.emit context ~topic:Service.manager_topic (Service.Client_state_changed manager); Ok ()) @@ -5429,6 +5434,21 @@ let test_graph_picker_controls scenario () = ; dispatcher_capacity = 256; wire_batch_max_bytes = 262144 } ; generation = "g"; projection_revision = "p" } | V2_list_journals _ -> P.V2_journals_outcome { items = []; next_cursor = None } + | V2_list_favorites _ when scenario = `Late_graph_cancel + && (Atomic.get current).snapshot.selected_graph = Some first -> + Fun.protect ~finally:(fun () -> + Atomic.set graph_read_cancelling true; + Eio.Cancel.protect (fun () -> + while not (Atomic.get release_graph_terminal) do Eio.Fiber.yield () done); + Atomic.set graph_read_finished true) (fun () -> + Atomic.set graph_read_worker (Some (W.Request_context.request_id context)); + Atomic.set graph_read_started true; + while not (Atomic.get release_graph_read) do Eio.Fiber.yield () done; + P.V2_favorites_outcome { favorites_page = None; generation = "g" + ; projection_revision = "p"; items = []; next_cursor = None }) + | V2_list_favorites _ when scenario = `Late_graph_cancel -> P.V2_favorites_outcome + { favorites_page = None; generation = "g" + ; projection_revision = "p"; items = []; next_cursor = None } | V2_get_asset_descriptors _ | V2_list_assets _ -> P.V2_assets_outcome { generation = "g"; projection_revision = "p"; items = []; next_cursor = None } | _ -> P.V2_failed { code = "UnusedFixtureRead"; message = "Unused graph-picker fixture read" } in @@ -5443,7 +5463,10 @@ let test_graph_picker_controls scenario () = W.on_event value (function | W.Response { outcome = Completed (Service.Local_copy_deleted _); _ } -> deletion_terminal := true - | W.Response { outcome = Completed (Service.Local_copy_inspected _) | Cancelled; _ } -> + | W.Response { request_id; outcome = Cancelled; _ } -> + incr inspection_terminals; + if Atomic.get graph_read_worker = Some request_id then incr graph_read_cancellations + | W.Response { outcome = Completed (Service.Local_copy_inspected _); _ } -> incr inspection_terminals | _ -> ())) service in let props = Hashtbl.create 128 in @@ -5485,10 +5508,16 @@ let test_graph_picker_controls scenario () = Fun.protect ~finally:(fun () -> Atomic.set finish_deletion true; Atomic.set release_automatic_inspection true; + Atomic.set release_graph_read true; + Atomic.set release_graph_terminal true; ignore (hooks.dispose ()); Option.iter Logseq_db_worker_lui.Journal_worker_runtime.stop !client) (fun () -> consume (hooks.init 2 2 config); wait "Switch graph action mounts" (fun () -> Option.is_some (find "text" "Switch graph")); + if scenario = `Late_graph_cancel then ( + wait "Favorites action mounts" (fun () -> Option.is_some (find "accessibility-label" "Favorites")); + press (Option.get (find "accessibility-label" "Favorites")); + wait "old graph Favorite read is genuinely in flight" (fun () -> Atomic.get graph_read_started)); press (Option.get (find "text" "Switch graph")); wait "graph list mounts" (fun () -> Option.is_some (find "accessibility-identifier" "graph-picker-list")); if scenario = `Late_presence then ( @@ -5530,6 +5559,22 @@ let test_graph_picker_controls scenario () = press (node ("graph-picker:" ^ Logseq_db_types.Graph_types.Uuid.to_string second)); wait "name selection reaches worker" (fun () -> List.exists (function Service.Select_graph id -> id = second | _ -> false) (Atomic.get commands)) + | `Late_graph_cancel -> + press (node ("graph-picker:" ^ Logseq_db_types.Graph_types.Uuid.to_string second)); + wait "new graph is visible while old cancellation cleanup is held" (fun () -> + Atomic.get graph_read_cancelling + && Atomic.get graph_opens >= 2 + && Option.is_some (find "accessibility-label" "Favorites")); + Alcotest.(check int) "old cancellation has not been delivered yet" 0 !graph_read_cancellations; + press (Option.get (find "accessibility-label" "Favorites")); + Atomic.set release_graph_terminal true; + wait "real old Worker cancellation is delivered after scope switch" (fun () -> + Atomic.get graph_read_finished && !graph_read_cancellations > 0); + consume (hooks.pump ()); + Alcotest.(check bool) "selected scope is current" true + ((Atomic.get current).snapshot.selected_graph = Some second); + Alcotest.(check bool) "old graph cancellation does not publish a current sync error" true + (Option.is_none (find "text" "Worker unavailable")) | (`Delete_menu | `Cancel | `Confirm | `Reopen | `Failure_select | `Presence_failure | `Refresh_after_presence_failure | `During_delete | `Late_presence) -> let stable_select = node ("graph-picker:" ^ Logseq_db_types.Graph_types.Uuid.to_string second) in if scenario = `Failure_select then retained_select := Some stable_select; @@ -5635,6 +5680,8 @@ let () = Alcotest.test_case "current marker and long names" `Quick (test_graph_picker_controls `Current); Alcotest.test_case "close retains current session" `Quick (test_graph_picker_controls `Close); Alcotest.test_case "name switches graph" `Quick (test_graph_picker_controls `Select); + Alcotest.test_case "old graph cancellation cannot publish a new-scope error" `Quick + (test_graph_picker_controls `Late_graph_cancel); Alcotest.test_case "row deletion confirmation" `Quick (test_graph_picker_controls `Delete_menu); Alcotest.test_case "current row deletion protection" `Quick (test_graph_picker_controls `Current_menu); Alcotest.test_case "cancel deletion keeps local copy" `Quick (test_graph_picker_controls `Cancel);