From 41291027920733f6637e765a87b7ca2af35f1fca Mon Sep 17 00:00:00 2001 From: pasichDev Date: Sat, 5 Sep 2026 00:22:24 +0300 Subject: [PATCH 1/4] fix(sync): close the data-loss paths found in the second review A verified review of the 2.6.50 sync code found ten correctness defects and thirty-one more beyond the ranking cap. The serious ones were races that a single device cannot show: - A note carrying attachments hashed differently on every device because its editor blocks were rewritten to device-local paths before hashing, so it conflicted with itself on every sync. Blocks now travel in a device-independent wire form and are mapped back on arrival. - A conflict was stored even for a record whose apply had just been skipped as stale, so resolving it later overwrote an edit the user made during the sync with a version they were never shown. - Older unresolved conflicts for the same record were never retired, so a stale pre-selected winner could be applied over the current edit. - Settings changed during a long sync were committed over and the hash rewritten to hide it. - A task, category or tag deleted here and edited elsewhere never came back: the update DAOs are no-ops on a missing row. - Restoring a ZIP extracted straight into live note folders before the JSON was validated; it now stages and adopts per note. - Applying a restored theme recreated BackupActivity mid-restore and disposed the in-flight inserts. - Reading history retained every bundle's bytes and validated ancestry recursively; superseded bundles are now pruned after a grace period. The rest: five digest implementations became one, three Drive failure classifiers became one, folder listings and attachment hashes are cached per sync, restore batches that repeat an id keep both rows, and same-named tags created on two devices reconcile deterministically. 308 unit tests (was 248) and 74 instrumentation tests (was 60), 0 failures; lint 0 errors; R8 clean. --- .github/workflows/ci.yml | 5 + .../pasich/mynotes/db/RoomSyncStoreTest.java | 555 +++++++++++- .../mynotes/base/activity/BaseActivity.java | 2 +- .../mynotes/data/database/dao/NoteDao.java | 4 + .../data/database/dao/SyncConflictDao.java | 15 + .../data/database/dao/SyncMetadataDao.java | 6 + .../mynotes/data/database/dao/TagsDao.java | 3 + .../preferences/AppPreferencesHelper.java | 47 +- .../data/sync/AttachmentHashCache.java | 149 ++++ .../mynotes/data/sync/AttachmentWireUrl.java | 44 + .../data/sync/GoogleDriveSyncBackend.java | 838 +++++++++++------- .../sync/PreferencesBaselineDecision.java | 60 ++ .../mynotes/data/sync/RoomSyncStore.java | 836 +++++++++++------ .../com/pasich/mynotes/data/sync/Sha256.java | 77 ++ .../data/sync/SnapshotBuildResult.java | 4 +- .../mynotes/data/sync/SnapshotProblem.java | 37 + .../pasich/mynotes/data/sync/SyncBackend.java | 63 +- .../mynotes/data/sync/SyncBundleCodec.java | 172 +++- .../data/sync/SyncBundleValidator.java | 67 +- .../data/sync/SyncMutationCoordinator.java | 103 ++- .../pasich/mynotes/data/sync/SyncRecord.java | 52 +- .../pasich/mynotes/data/sync/SyncService.java | 198 ++--- .../pasich/mynotes/data/sync/SyncStore.java | 19 +- .../data/sync/VerifyingInputStream.java | 147 +++ .../attach/AttachmentStorage.java | 13 + .../extendedEditor/attach/AttachmentUrl.java | 10 +- .../attach/EditorAttachmentBlocks.java | 146 +++ .../attach/NoteAttachmentRelocator.java | 178 +++- .../extendedEditor/utils/EditorJsonUtils.java | 41 +- .../ui/view/activity/BackupActivity.java | 49 +- .../utils/backup/local/ZipBackupHelper.java | 155 ++-- .../preferences/AppPreferencesHelperTest.java | 55 ++ .../data/sync/AttachmentHashCacheTest.java | 104 +++ .../data/sync/AttachmentWireUrlTest.java | 31 + .../data/sync/GoogleDriveSyncBackendTest.java | 372 +++++++- .../sync/PreferencesBaselineDecisionTest.java | 51 ++ .../pasich/mynotes/data/sync/Sha256Test.java | 59 ++ .../data/sync/SnapshotProblemTest.java | 59 ++ .../data/sync/SyncBundleCodecTest.java | 106 +++ .../data/sync/SyncConvergenceTest.java | 8 +- .../sync/SyncMutationCoordinatorTest.java | 124 ++- .../mynotes/data/sync/SyncRecordTest.java | 45 + .../mynotes/data/sync/SyncServiceTest.java | 88 +- .../data/sync/VerifyingInputStreamTest.java | 103 +++ .../attach/EditorAttachmentBlocksTest.java | 84 ++ .../attach/NoteAttachmentRelocatorTest.java | 95 ++ .../backup/local/ZipBackupHelperTest.java | 103 +++ 47 files changed, 4493 insertions(+), 1089 deletions(-) create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java create mode 100644 app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95f06eb4..b4c898b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,7 +187,12 @@ jobs: # Posts, and on later pushes updates, a single comment showing overall coverage and the # coverage of the files this PR actually changed. Both reports are passed together so the # numbers reflect the unit and on-device suites combined. + # + # Skipped for a pull request from a fork: its GITHUB_TOKEN is read-only whatever the + # workflow asks for, so the comment call would fail with a 403 and turn the whole job red for + # a reason unrelated to the change. - name: Comment coverage + if: github.event.pull_request.head.repo.full_name == github.repository uses: madrapps/jacoco-report@v1.7.1 with: paths: | diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index 6f1d88db..83afa855 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -294,10 +294,11 @@ public void applySnapshot_repointsEditorBlocksAtTheFilesThisDeviceWrote() throws } @Test - public void applySnapshot_leavesEditorBlocksAloneWhenTheyDoNotLineUp() throws Exception { + public void applySnapshot_rewritesOnlyTheBlocksThatNameAKnownAttachment() throws Exception { byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); int noteId = seedNoteWithAttachment("photo.png", bytes); - // Two blocks, one attachment: the positional mapping cannot be trusted. + // Two blocks, one attachment: a block naming a file the column does not know is left + // exactly as it was, and the one that does is repointed by identity, not by position. String twoBlocks = "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_" + noteId @@ -311,8 +312,530 @@ public void applySnapshot_leavesEditorBlocksAloneWhenTheyDoNotLineUp() throws Ex store.applySnapshot(store.readSnapshot(), Collections.emptyList()); + List urls = + com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks.fileUrls( + db.noteDao().getNoteSync(noteId).getValueJson()); + assertThat(urls).hasSize(2); + File rendered = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve( + context, urls.get(0)); + assertThat(rendered).isNotNull(); + assertThat(readAll(new java.io.FileInputStream(rendered))).isEqualTo(bytes); + assertThat(urls.get(1)).isEqualTo("editorjs://attachments/note_" + noteId + "/other.png"); + } + + @Test + public void applySnapshot_leavesLegacyBlocksAloneWhenTheyDoNotLineUp() throws Exception { + // A bundle from an older client names the sender's files in its blocks; for those the + // only mapping is positional, and two blocks for one attachment cannot be trusted. // Rewriting on a guess could point a block at the wrong file; leaving it is recoverable. - assertThat(db.noteDao().getNoteSync(noteId).getValueJson()).isEqualTo(twoBlocks); + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("photo.png", bytes); + SyncRecord built = onlyNote(store.readSnapshot()); + String legacyBlocks = + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_99/photo.png\"}}}," + + "{\"type\":\"image\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_99/other.png\"}}}]"; + JsonObject legacyPayload = built.getPayload(); + legacyPayload.addProperty("f", legacyBlocks); + SyncRecord legacy = + SyncRecord.live( + SyncRecord.Type.NOTE, + built.getId(), + built.getUpdatedAt().plusSeconds(1), + legacyPayload); + + store.applySnapshot( + new SyncSnapshot(Collections.singletonList(legacy)), Collections.emptyList()); + + assertThat(db.noteDao().getNoteSync(noteId).getValueJson()).isEqualTo(legacyBlocks); + } + + // ------------------------------------------------- a note that has crossed devices + + @Test + public void aNoteReceivedFromAnotherDeviceHashesIdenticallyWhenRebuiltThere() throws Exception { + // Device A: a rich note whose block names A's own file. A throwaway note first, so A's + // row id differs from the one B will assign. + seedNote("Placeholder", "x", null, "22222222-2222-4222-8222-222222222222"); + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + // The display name has no extension, so the MIME type can only come from the sender's + // file name — which B never sees. + int noteId = seedNoteWithAttachment("1700000000000_123.png", "photo", bytes); + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, "1700000000000_123.png") + + "\",\"name\":\"photo\"}}}]"); + db.noteDao().addNote(seeded); + SyncRecord fromA = + store.readSnapshot() + .find(SyncRecord.Type.NOTE, "11111111-1111-4111-8111-111111111111"); + assertThat(fromA).isNotNull(); + + // On the wire the block names the attachment, not A's row id and file. + String wireBlocks = fromA.getPayload().get("f").getAsString(); + assertThat(wireBlocks).doesNotContain("note_"); + assertThat(wireBlocks).contains("mynotes-sync://attachment/"); + + // Device B: its own database, its own row ids; the blob arrives through the sync cache. + AppDatabase dbB = + Room.inMemoryDatabaseBuilder(context, AppDatabase.class) + .allowMainThreadQueries() + .build(); + try { + RoomSyncStore storeB = new RoomSyncStore(context, dbB, mock(PreferenceHelper.class)); + storeB.writeAttachment(sha256(bytes), bytes.length, new ByteArrayInputStream(bytes)); + storeB.applySnapshot( + new SyncSnapshot(Collections.singletonList(fromA)), Collections.emptyList()); + + SyncRecord rebuiltOnB = onlyNote(storeB.readSnapshot()); + + // The same version, not a conflict against itself: restoring rewrote B's blocks and + // column to B's files, and every such note used to hash differently at the same + // timestamp and republish a bundle on every sync forever. + assertThat(rebuiltOnB.getCanonicalPayloadHash()) + .isEqualTo(fromA.getCanonicalPayloadHash()); + long localIdOnB = + dbB.syncMetadataDao() + .getByStableId(SyncMetadata.RECORD_TYPE_NOTE, fromA.getId()) + .localId; + String blockUrl = + com.pasich + .mynotes + .extendedEditor + .attach + .EditorAttachmentBlocks + .fileUrls(dbB.noteDao().getNoteSync((int) localIdOnB).getValueJson()) + .get(0); + File rendered = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve( + context, blockUrl); + assertThat(rendered).isNotNull(); + assertThat(readAll(new java.io.FileInputStream(rendered))).isEqualTo(bytes); + } finally { + dbB.close(); + } + } + + // ------------------------------------------------- conflicts and records that moved on + + @Test + public void applySnapshot_doesNotStoreAConflictForARecordSkippedAsStale() throws Exception { + int noteId = seedNote("Edited during the sync", "body", null); + SyncRecord local = onlyNote(store.readSnapshot()); + // The user edits while the sync is on Drive. + db.syncMetadataDao().touch(SyncMetadata.RECORD_TYPE_NOTE, noteId, 5_000L); + JsonObject remotePayload = local.getPayload(); + remotePayload.addProperty("b", "Remote title"); + SyncRecord remote = + SyncRecord.live( + SyncRecord.Type.NOTE, + local.getId(), + java.time.Instant.ofEpochMilli(2_000L), + remotePayload); + com.pasich.mynotes.data.sync.SyncMergeResult merge = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(local)), + new SyncSnapshot(Collections.singletonList(remote))); + assertThat(merge.getConflicts()).hasSize(1); + + store.applySnapshot(merge.getMergedSnapshot(), merge.getConflicts()); + + // The apply rightly skipped the note; the conflict it would have stored offered two + // versions older than the edit, and resolving it wrote one of them over the edit. + assertThat(store.getConflicts()).isEmpty(); + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Edited during the sync"); + } + + @Test + public void applySnapshot_retiresAnOpenConflictWhoseWinnerIsNoLongerTheLiveVersion() + throws Exception { + int noteId = seedNote("Current", "body", null); + SyncRecord local = onlyNote(store.readSnapshot()); + db.syncConflictDao() + .insertIgnoringDuplicates( + Collections.singletonList( + noteConflictRow(local.getId(), "old-winner", 900L, 800L))); + JsonObject remotePayload = local.getPayload(); + remotePayload.addProperty("b", "Remote title"); + SyncRecord remote = + SyncRecord.live( + SyncRecord.Type.NOTE, + local.getId(), + java.time.Instant.ofEpochMilli(2_000L), + remotePayload); + com.pasich.mynotes.data.sync.SyncMergeResult merge = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(local)), + new SyncSnapshot(Collections.singletonList(remote))); + + store.applySnapshot(merge.getMergedSnapshot(), merge.getConflicts()); + + // The old row pre-selected a winner the record has moved past; one tap on it reverted + // the live version. Only the conflict against the current version remains. + List unresolved = + store.getUnresolvedConflicts(); + assertThat(unresolved).hasSize(1); + assertThat(unresolved.get(0).winnerVersionId).isEqualTo(remote.getCanonicalPayloadHash()); + assertThat(noteId).isGreaterThan(0); + } + + @Test + public void resolveConflict_dropsAConflictTheRecordHasMovedPastInsteadOfApplyingIt() + throws Exception { + int noteId = seedNote("Newest", "body", null); + db.syncMetadataDao().touch(SyncMetadata.RECORD_TYPE_NOTE, noteId, 5_000L); + db.syncConflictDao() + .insertIgnoringDuplicates( + Collections.singletonList( + noteConflictRow( + "11111111-1111-4111-8111-111111111111", + "stale-winner", + 3_000L, + 2_000L))); + long conflictId = db.syncConflictDao().getAll().get(0).id; + + store.resolveConflict(conflictId, SyncResolution.KEEP_WINNER); + + // Both offered versions are older than what the user has now; applying either would + // have overwritten the edit with a version that was never offered against it. + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Newest"); + assertThat(db.syncConflictDao().getById(conflictId)).isNull(); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_NOTE, noteId).updatedAt) + .isEqualTo(5_000L); + } + + /** A stored note conflict whose winner is titled after its version id. */ + private com.pasich.mynotes.data.database.entities.SyncConflictEntity noteConflictRow( + String stableId, String winnerVersionId, long winnerUpdatedAt, long loserUpdatedAt) { + String winner = + "{\"type\":\"note\",\"id\":\"" + + stableId + + "\",\"updatedAt\":\"" + + java.time.Instant.ofEpochMilli(winnerUpdatedAt) + + "\",\"deletedAt\":null,\"payload\":{\"b\":\"" + + winnerVersionId + + "\",\"c\":\"body\"}}"; + String loser = + "{\"type\":\"note\",\"id\":\"" + + stableId + + "\",\"updatedAt\":\"" + + java.time.Instant.ofEpochMilli(loserUpdatedAt) + + "\",\"deletedAt\":null,\"payload\":{\"b\":\"loser\",\"c\":\"body\"}}"; + return new com.pasich.mynotes.data.database.entities.SyncConflictEntity( + SyncMetadata.RECORD_TYPE_NOTE, + stableId, + "pair-" + winnerVersionId, + "REMOTE", + "LOCAL", + winnerVersionId, + "loser-" + winnerVersionId, + winner, + loser, + winnerUpdatedAt, + loserUpdatedAt, + false, + false, + "PENDING", + false, + 1L, + 0L); + } + + // ------------------------------------------------- settings edited during a sync + + @Test + public void applySnapshot_leavesSettingsChangedDuringTheSyncAlone() throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + adapter.current.set(preferencesWithTheme(1)); + preferencesStore.buildSnapshot(); + // The user flips a setting while the sync is on Drive; the settings screens write the + // preferences directly and nothing touches the sync record for them. + adapter.current.set(preferencesWithTheme(2)); + db.syncMetadataDao().setVersion(SyncMetadata.RECORD_TYPE_PREFERENCES, 0, 1_000L, null); + SyncRecord remote = + SyncRecord.live( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000", + java.time.Instant.ofEpochMilli(2_000L), + new com.google.gson.Gson() + .toJsonTree(preferencesWithTheme(3)) + .getAsJsonObject()); + + preferencesStore.applySnapshot( + new SyncSnapshot(Collections.singletonList(remote)), Collections.emptyList()); + + // Not committed, and the baseline not rewritten to hide it: the merged version was chosen + // against settings that no longer exist. + assertThat(adapter.committed.get()).isNull(); + assertThat(adapter.current.get().getThemeValue()).isEqualTo(2); + // The next build sees the edit and publishes it. + preferencesStore.buildSnapshot(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000") + .updatedAt) + .isGreaterThan(2_000L); + } + + @Test + public void applySnapshot_skipsAnUnusablePreferencesPayloadInsteadOfFailingEverySync() + throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + int noteId = seedNote("Applied anyway", "body", null); + SyncRecord note = onlyNote(preferencesStore.readSnapshot()); + JsonObject changed = note.getPayload(); + changed.addProperty("b", "Remote title"); + SyncRecord remoteNote = + SyncRecord.live( + SyncRecord.Type.NOTE, + note.getId(), + java.time.Instant.ofEpochMilli(2_000L), + changed); + // A payload with no "g": nothing this app ever wrote, but one such record on Drive used + // to stop every device from syncing anything until someone changed a setting locally. + SyncRecord invalidPreferences = + SyncRecord.live( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000", + java.time.Instant.ofEpochMilli(2_000L), + new JsonObject()); + + preferencesStore.applySnapshot( + new SyncSnapshot(java.util.Arrays.asList(remoteNote, invalidPreferences)), + Collections.emptyList()); + + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Remote title"); + assertThat(adapter.committed.get()).isNull(); + } + + @Test + public void clearAfterDisconnect_dropsThePendingPreferencesJournal() throws Exception { + db.syncPendingPreferencesDao() + .upsert( + new com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity( + 1, "{}", "target", "baseline", 0L, false, 0L, "")); + + store.clearAfterDisconnect(); + + // Left behind, a fresh store replayed the disconnected account's settings onto the + // device at its next seeding. + assertThat(db.syncPendingPreferencesDao().get()).isNull(); + assertThat(db.syncPendingPreferencesDao().getIncludingQuarantined()).isNull(); + } + + // ------------------------------------------------- records deleted here, edited elsewhere + + @Test + public void applySnapshot_bringsBackATaskDeletedHereAndEditedElsewhere() throws Exception { + com.pasich.mynotes.data.model.Task task = new com.pasich.mynotes.data.model.Task("Call", 0); + int taskId = (int) db.taskDao().insertTask(task); + String stableId = "33333333-3333-4333-8333-333333333333"; + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TASK, taskId, stableId, 1_000L, null)); + db.taskDao().deleteById(taskId); + db.syncMetadataDao().markDeleted(SyncMetadata.RECORD_TYPE_TASK, taskId, 1_500L); + JsonObject edited = new com.google.gson.Gson().toJsonTree(task).getAsJsonObject(); + edited.addProperty("title", "Call back"); + SyncMetadata.stripDeviceLocalFields(SyncMetadata.RECORD_TYPE_TASK, edited); + + store.applySnapshot( + new SyncSnapshot( + Collections.singletonList( + SyncRecord.live( + SyncRecord.Type.TASK, + stableId, + java.time.Instant.ofEpochMilli(3_000L), + edited))), + Collections.emptyList()); + + // @Update on the deleted row was a silent no-op while the tombstone was cleared anyway, + // so the task never came back here and the remote edit was re-applied to nothing forever. + com.pasich.mynotes.data.model.Task revived = db.taskDao().getTaskSync(taskId); + assertThat(revived).isNotNull(); + assertThat(revived.getTitle()).isEqualTo("Call back"); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TASK, taskId).deletedAt) + .isNull(); + } + + // ------------------------------------------------- tags created by name on two devices + + @Test + public void applySnapshot_reconcilesATagCreatedUnderTheSameNameOnAnotherDevice() + throws Exception { + com.pasich.mynotes.data.model.Tag local = + new com.pasich.mynotes.data.model.Tag().create("Work"); + long localId = db.tagsDao().addTag(local); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + localId, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + 1_000L, + null)); + + store.applySnapshot( + new SyncSnapshot( + Collections.singletonList( + remoteTag("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "Work"))), + Collections.emptyList()); + + // One "Work", under the identity both devices will settle on; the other identity is + // tombstoned so the next sync retires it everywhere instead of leaving two rows. + assertThat(tagsNamed("Work")).isEqualTo(1); + SyncMetadataEntity winner = + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_TAG, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); + assertThat(winner).isNotNull(); + assertThat(winner.deletedAt).isNull(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_TAG, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb") + .deletedAt) + .isNotNull(); + } + + @Test + public void applySnapshot_keepsTheLocalTagWhenItHoldsTheWinningIdentity() throws Exception { + com.pasich.mynotes.data.model.Tag local = + new com.pasich.mynotes.data.model.Tag().create("Work"); + long localId = db.tagsDao().addTag(local); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + localId, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + 1_000L, + null)); + + store.applySnapshot( + new SyncSnapshot( + Collections.singletonList( + remoteTag("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "Work"))), + Collections.emptyList()); + + assertThat(tagsNamed("Work")).isEqualTo(1); + assertThat(db.tagsDao().getTagSync(localId)).isNotNull(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_TAG, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")) + .isNull(); + } + + private static SyncRecord remoteTag(String stableId, String name) { + JsonObject payload = new JsonObject(); + payload.addProperty("b", name); + payload.addProperty("c", 0); + payload.addProperty("d", 0); + payload.addProperty("e", -1); + return SyncRecord.live( + SyncRecord.Type.TAG, stableId, java.time.Instant.ofEpochMilli(1_000L), payload); + } + + private int tagsNamed(String name) { + int count = 0; + for (com.pasich.mynotes.data.model.Tag tag : db.tagsDao().getTags().blockingFirst()) { + if (name.equals(tag.getNameTag())) count++; + } + return count; + } + + // ------------------------------------------------- blobs a fresh store has to find + + @Test + public void hasAttachment_findsABlobInANoteFolderWithoutHavingBuiltASnapshot() + throws Exception { + byte[] bytes = "owned bytes".getBytes(StandardCharsets.UTF_8); + seedNoteWithAttachment("owned.png", bytes); + + // The Backup screen's store, resolving a conflict the worker's store found: it has never + // built a snapshot, and the only index of note-folder files used to be built there. + RoomSyncStore fresh = new RoomSyncStore(context, db, mock(PreferenceHelper.class)); + + assertThat(fresh.hasAttachment(sha256(bytes))).isTrue(); + } + + // ------------------------------------------------- attachments whose file is gone + + @Test + public void buildSnapshot_describesAMissingFileFromWhatTheColumnRemembersAndRestoresIt() + throws Exception { + byte[] bytes = "restored later".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + String logicalId = "7d444840-9dc0-11d1-b245-5ffdce74fad2"; + int noteId = seedNote("Repairable", "body", null); + // The column a sync wrote: the file it names is gone, but the id, hash, size and type + // are all there — enough to publish the note and to fetch the bytes back. + Note note = db.noteDao().getNoteSync(noteId); + note.setAttachments( + "[{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, logicalId + "-" + hash) + + "\",\"name\":\"photo.png\",\"id\":\"" + + logicalId + + "\",\"sha256\":\"" + + hash + + "\",\"size\":" + + bytes.length + + ",\"mimeType\":\"image/png\"}]"); + db.noteDao().addNote(note); + // The blob is in the sync cache, as SyncService puts it after downloading from Drive. + store.writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + + SnapshotBuildResult result = store.buildSnapshot(); + + // Every sync used to end in MISSING_ATTACHMENT here, with no way back but deleting the + // block by hand. + assertThat(result.isPublishable()).isTrue(); + JsonObject payload = onlyNote(result.requireSnapshot()).getPayload(); + assertThat(payload.getAsJsonArray("attachmentHashes").get(0).getAsString()).isEqualTo(hash); + assertThat( + payload.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .get("mimeType") + .getAsString()) + .isEqualTo("image/png"); + + store.applySnapshot(result.requireSnapshot(), Collections.emptyList()); + + File repaired = resolveFirstAttachment(db.noteDao().getNoteSync(noteId).getAttachments()); + assertThat(repaired.isFile()).isTrue(); + assertThat(readAll(new java.io.FileInputStream(repaired))).isEqualTo(bytes); + } + + @Test + public void buildSnapshot_namesTheNoteWhoseAttachmentCannotBeFound() { + int noteId = seedNote("Shopping list", "body", null); + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setAttachments("[" + attachmentJson(noteId, "gone.png") + "]"); + db.noteDao().addNote(seeded); + + SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(store); + + // The account screen shows this string; "MISSING_ATTACHMENT" alone left the user to + // guess which note to open. + assertThat(error).hasMessageThat().contains("note \"Shopping list\""); + assertThat(error.getProblems().get(0).getLabel()).isEqualTo("Shopping list"); } /** Resolves the first entry of an attachments JSON the way the app's consumers do. */ @@ -460,17 +983,17 @@ record -> { // ---- helpers ---- private int seedNote(String title, String value, String attachmentsJson) { + return seedNote(title, value, attachmentsJson, "11111111-1111-4111-8111-111111111111"); + } + + private int seedNote(String title, String value, String attachmentsJson, String stableId) { Note note = new Note().create(title, value, 1_000L, ""); note.setAttachments(attachmentsJson); int id = db.noteDao().addNote(note).intValue(); db.syncMetadataDao() .insertIfAbsent( new SyncMetadataEntity( - SyncMetadata.RECORD_TYPE_NOTE, - id, - "11111111-1111-4111-8111-111111111111", - 1_000L, - null)); + SyncMetadata.RECORD_TYPE_NOTE, id, stableId, 1_000L, null)); return id; } @@ -775,6 +1298,11 @@ private static com.pasich.mynotes.utils.backup.models.PreferencesBackup preferen /** Writes a real file into the note's own attachment folder and links it from the note. */ private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOException { + return seedNoteWithAttachment(fileName, fileName, bytes); + } + + private int seedNoteWithAttachment(String fileName, String displayName, byte[] bytes) + throws IOException { int id = seedNote("With attachment", "body", null); File folder = new File(context.getFilesDir(), "attachments/note_" + id); assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); @@ -782,7 +1310,7 @@ private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOExcep out.write(bytes); } // Production shape: EditorJSInterface writes editorjs://attachments/note_/. - String json = "[" + attachmentJson(id, fileName) + "]"; + String json = "[" + attachmentJson(id, fileName, displayName) + "]"; Note note = db.noteDao().getNoteSync(id); note.setAttachments(json); db.noteDao().addNote(note); @@ -809,10 +1337,15 @@ private static SnapshotBuildResult.SnapshotBuildException assertSnapshotBuildFai /** The canonical reference the editor and sync restore both produce. */ private static String attachmentJson(int noteId, String name) { + return attachmentJson(noteId, name, name); + } + + private static String attachmentJson(int noteId, String fileName, String displayName) { return "{\"url\":\"" - + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor(noteId, name) + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, fileName) + "\",\"name\":\"" - + name + + displayName + "\"}"; } diff --git a/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java b/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java index 7908e59c..de0cc9b4 100644 --- a/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java +++ b/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java @@ -31,7 +31,7 @@ public abstract class BaseActivity extends AppCompatActivity implements BaseView { - @Inject ThemePreferencesCache themePreferencesCache; + @Inject protected ThemePreferencesCache themePreferencesCache; @Override public void selectTheme() { diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java index 46a721d1..ff0a6198 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java @@ -29,6 +29,10 @@ public interface NoteDao { @Query("SELECT * FROM notes WHERE id = :id LIMIT 1") Note getNoteSync(int id); + /** One round trip for a whole restore batch; the caller keeps the list under the bind limit. */ + @Query("SELECT * FROM notes WHERE id IN (:ids)") + List getNotesByIdsSync(List ids); + @Query("DELETE FROM notes WHERE id = :id") void deleteById(int id); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java index 24983967..8c5fb51f 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java @@ -36,6 +36,21 @@ public interface SyncConflictDao { @Query("SELECT * FROM sync_conflicts WHERE id = :id LIMIT 1") SyncConflictEntity getById(long id); + @Query("DELETE FROM sync_conflicts WHERE id = :id") + void deleteById(long id); + + /** + * Retires every open conflict for a record whose winner is no longer the version being applied. + * + *

Such a row offers, pre-selected, a version the record has since moved past; applying it + * reverted the newer edit. The loser it carried is republished with the bundle and comes back + * against the current version. + */ + @Query( + "DELETE FROM sync_conflicts WHERE resolved = 0 AND recordType = :recordType " + + "AND stableId = :stableId AND winnerVersionId != :winnerVersionId") + void deleteSupersededUnresolved(String recordType, String stableId, String winnerVersionId); + @Query( "UPDATE sync_conflicts SET resolution = :resolution, resolved = 1, resolvedAt = :resolvedAt WHERE id = :id") void markResolved(long id, String resolution, long resolvedAt); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java index 26821015..80ab8197 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java @@ -28,6 +28,12 @@ public interface SyncMetadataDao { + "WHERE recordType = :recordType AND localId = :localId)") boolean exists(String recordType, long localId); + /** The subset of {@code localIds} that already has a row; the caller chunks the list. */ + @Query( + "SELECT localId FROM sync_metadata " + + "WHERE recordType = :recordType AND localId IN (:localIds)") + List getExistingLocalIds(String recordType, List localIds); + @Query( "SELECT * FROM sync_metadata WHERE recordType = :recordType AND stableId = :stableId LIMIT 1") SyncMetadataEntity getByStableId(String recordType, String stableId); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java index 8ee0ac99..9f19484f 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java @@ -20,6 +20,9 @@ public interface TagsDao { @Query("SELECT * FROM tags WHERE id = :id LIMIT 1") Tag getTagSync(long id); + @Query("SELECT * FROM tags WHERE id IN (:ids)") + List getTagsByIdsSync(List ids); + /** Tags are referenced by name from a note, so the name is their real identity. */ @Query("SELECT * FROM tags WHERE name = :name LIMIT 1") Tag getTagByNameSync(String name); diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java index d62f5f80..93c839ba 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java @@ -15,13 +15,28 @@ public class AppPreferencesHelper implements PreferenceHelper { private final ThemePreferencesCache themeCache; private final SafePreferences prefs; + private final java.util.concurrent.Executor mainThread; @Inject AppPreferencesHelper( AppPreferencesCache appCache, ThemePreferencesCache themeCache, SafePreferences prefs) { + this( + appCache, + themeCache, + prefs, + new android.os.Handler(android.os.Looper.getMainLooper())::post); + } + + /** Test seam: where the theme application is posted to. */ + AppPreferencesHelper( + AppPreferencesCache appCache, + ThemePreferencesCache themeCache, + SafePreferences prefs, + java.util.concurrent.Executor mainThread) { this.prefs = prefs; this.appCache = appCache; this.themeCache = themeCache; + this.mainThread = mainThread; this.appCache.initialize(); this.themeCache.initialize(); } @@ -88,10 +103,19 @@ public PreferencesBackup getListPreferences() { PreferencesConfig.ARGUMENT_DEFAULT_UI_SCALING_VALUE)); } - /** Persists all fields from a backup and refreshes the caches. */ + /** + * Persists all fields from a backup and refreshes the caches. + * + *

The restore path. The theme is deliberately not applied here: {@code + * AppCompatDelegate.setDefaultNightMode} recreates every started activity when the mode + * changes, and this runs at the start of a restore whose note and tag inserts are still in + * flight on the Backup screen — recreating it disposed those inserts and left the database half + * restored with no message. The stored mode takes effect at the next activity creation, and the + * screen applies it itself once the restore has finished. + */ @Override public void setListPreferences(PreferencesBackup preferences) { - commitListPreferences(preferences); + commitListPreferences(preferences, false); } /** @@ -103,10 +127,16 @@ public void setListPreferences(PreferencesBackup preferences) { * plus {@code commit()} makes the whole set atomic and tells the caller whether it is durable, * which is what lets {@code RoomSyncStore} decide when the journal may be dropped. * + *

The sync path: a theme arriving from another device is applied at once. + * * @return true when the values are durably stored, false when the write failed. */ @Override public boolean commitListPreferences(PreferencesBackup preferences) { + return commitListPreferences(preferences, true); + } + + private boolean commitListPreferences(PreferencesBackup preferences, boolean applyThemeNow) { if (preferences == null || !preferences.isCreated()) { return false; } @@ -137,12 +167,13 @@ public boolean commitListPreferences(PreferencesBackup preferences) { } appCache.refresh(); themeCache.refresh(); - // Refreshing the caches only reloads the values. Light/dark is owned by - // AppCompatDelegate, which has to be told, or a theme arriving from another device sat - // in storage until the next activity was created. Posted to the main thread because this - // runs on a background thread for both a sync apply and a backup restore. - new android.os.Handler(android.os.Looper.getMainLooper()) - .post(themeCache::applyCurrentThemeMode); + if (applyThemeNow) { + // Refreshing the caches only reloads the values. Light/dark is owned by + // AppCompatDelegate, which has to be told, or a theme arriving from another device + // sat in storage until the next activity was created. Posted to the main thread + // because a sync apply runs on a background thread. + mainThread.execute(themeCache::applyCurrentThemeMode); + } return true; } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java new file mode 100644 index 00000000..cd418128 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java @@ -0,0 +1,149 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Remembers the hash of an attachment file until the file changes. + * + *

Every snapshot build used to read and digest every attachment in the library, and a sync + * builds at least once — twice when the first-sync estimate precedes it — so an idle sync of a + * library with gigabytes of attachments was dominated by re-hashing bytes nothing had touched. A + * file is keyed by its path, size and modification time, the same test a version control index + * uses; a file rewritten in place with identical size within the same millisecond is the one case + * this cannot see, and no writer in this app does that. + * + *

Persisted as JSON next to the download cache so the saving survives the store instance, which + * lives only as long as one sync. Loading, saving and every lookup is best effort: a lost or + * unreadable cache costs one full re-hash, never correctness. + */ +final class AttachmentHashCache { + + /** Produces the hash when the cache has no answer. */ + interface Hasher { + @NonNull + String sha256(@NonNull File file) throws IOException; + } + + private static final class Entry { + final long size; + final long modifiedAt; + final String sha256; + + Entry(long size, long modifiedAt, String sha256) { + this.size = size; + this.modifiedAt = modifiedAt; + this.sha256 = sha256; + } + } + + @Nullable private final File storage; + private final Map entries = new LinkedHashMap<>(); + private boolean loaded; + private boolean dirty; + + /** + * @param storage where the cache persists, or {@code null} to keep it in memory only. + */ + AttachmentHashCache(@Nullable File storage) { + this.storage = storage; + } + + /** The file's hash, from the cache when its size and modification time still match. */ + @NonNull + synchronized String sha256(@NonNull File file, @NonNull Hasher hasher) throws IOException { + load(); + String key = file.getAbsolutePath(); + long size = file.length(); + long modifiedAt = file.lastModified(); + Entry cached = entries.get(key); + if (cached != null && cached.size == size && cached.modifiedAt == modifiedAt) { + return cached.sha256; + } + String hash = hasher.sha256(file); + entries.put(key, new Entry(size, modifiedAt, hash)); + dirty = true; + return hash; + } + + /** Writes the cache if anything changed; a failure here is logged by nobody on purpose. */ + synchronized void flush() { + if (!dirty || storage == null) { + return; + } + JsonObject root = new JsonObject(); + for (Map.Entry entry : entries.entrySet()) { + JsonObject value = new JsonObject(); + value.addProperty("size", entry.getValue().size); + value.addProperty("modifiedAt", entry.getValue().modifiedAt); + value.addProperty("sha256", entry.getValue().sha256); + root.add(entry.getKey(), value); + } + File parent = storage.getParentFile(); + if (parent != null && !parent.isDirectory() && !parent.mkdirs()) { + return; + } + File temporary = new File(parent, storage.getName() + ".tmp"); + try { + Files.write(temporary.toPath(), root.toString().getBytes(StandardCharsets.UTF_8)); + Files.move( + temporary.toPath(), + storage.toPath(), + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + dirty = false; + } catch (IOException | RuntimeException ignored) { + // The next build simply hashes again. + temporary.delete(); + } + } + + /** Forgets everything, in memory and on disk. */ + synchronized void clear() { + entries.clear(); + loaded = true; + dirty = false; + if (storage != null) { + storage.delete(); + } + } + + private void load() { + if (loaded) { + return; + } + loaded = true; + if (storage == null || !storage.isFile()) { + return; + } + try { + String json = new String(Files.readAllBytes(storage.toPath()), StandardCharsets.UTF_8); + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + for (Map.Entry entry : root.entrySet()) { + JsonObject value = entry.getValue().getAsJsonObject(); + String sha256 = value.get("sha256").getAsString(); + if (!sha256.matches("[0-9a-f]{64}")) { + continue; + } + entries.put( + entry.getKey(), + new Entry( + value.get("size").getAsLong(), + value.get("modifiedAt").getAsLong(), + sha256)); + } + } catch (IOException | RuntimeException unreadable) { + entries.clear(); + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java new file mode 100644 index 00000000..ccb52b7a --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java @@ -0,0 +1,44 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.util.UUID; + +/** + * The device-independent form of an attachment reference inside a note's editor blocks. + * + *

A block on the device that wrote it names {@code editorjs://attachments/note_7/photo.png}: a + * Room row id and a file name that exist nowhere else. Restoring on another device necessarily + * rewrites it, so the same note hashed differently on every device that held it, and the merge + * reported a conflict against itself on every sync. On the wire a block therefore names the + * attachment's logical id, which is the one identity every device agrees on; each store maps it to + * its own file on the way in and back on the way out. + * + *

The scheme is deliberately one nothing else parses: if a wire reference ever leaked into a + * stored note it would render as a missing file rather than be mistaken for a local path. + */ +final class AttachmentWireUrl { + + private static final String PREFIX = "mynotes-sync://attachment/"; + + private AttachmentWireUrl() {} + + @NonNull + static String forLogicalId(@NonNull String logicalId) { + return PREFIX + logicalId; + } + + /** The logical id a wire reference names, or {@code null} for any other URL. */ + @Nullable + static String logicalIdOf(@Nullable String url) { + if (url == null || !url.startsWith(PREFIX)) { + return null; + } + String id = url.substring(PREFIX.length()); + try { + return UUID.fromString(id).toString().equals(id) ? id : null; + } catch (IllegalArgumentException notAUuid) { + return null; + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index 204f910e..f130a1b4 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -4,6 +4,7 @@ import androidx.annotation.Nullable; import com.google.gson.Gson; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -15,17 +16,17 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Clock; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -40,13 +41,28 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final String MIME_JSON = "application/json; charset=UTF-8"; private static final String MIME_ZIP = "application/zip"; private static final String MIME_BINARY = "application/octet-stream"; + private static final String PROPERTY_OWNER = "mynotesOwner"; + private static final String PROPERTY_BUNDLE = "mynotesBundle"; + private static final String PROPERTY_BUNDLE_PUBLISHED_AT = "mynotesBundlePublishedAt"; + private static final String PROPERTY_ATTACHMENT_SHA256 = "mynotesAttachmentSha256"; private static final int MAX_BUNDLE_RESPONSE_BYTES = 32 * 1024 * 1024; - private static final int MAX_ATTACHMENT_RESPONSE_BYTES = 100 * 1024 * 1024; + private static final long MAX_ATTACHMENT_RESPONSE_BYTES = + SyncBundleValidator.MAX_ATTACHMENT_BYTES; private static final int RESUMABLE_CHUNK_BYTES = 256 * 1024; private static final int HTTP_RESUME_INCOMPLETE = 308; private static final int MAX_STALLED_CHUNK_ATTEMPTS = 3; private static final int MAX_ERROR_DETAIL_BYTES = 1024; private static final int MAX_ERROR_DETAIL_CHARS = 200; + + /** + * How long a superseded bundle stays after its successor appears. + * + *

A device that listed the folder just before the successor was published may still be + * fetching the old bundle; an hour outlives any read, including the six-hourly worker's, which + * WorkManager stops after ten minutes. + */ + static final long BUNDLE_PRUNE_GRACE_MILLIS = 60L * 60L * 1000L; + private static final Gson GSON = new Gson(); private final String accessToken; @@ -56,17 +72,31 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private final SyncBundleCodec bundleCodec; private final DriveRequestExecutor requestExecutor; private final SyncMerger merger = new SyncMerger(); - private List lastReadFrontierBundleIds = Collections.emptyList(); + private final long maxAttachmentBytes; private String lastReadToken = ""; + /** Every bundle file the last read saw, so a publish can retire the ones it supersedes. */ + private List lastReadBundles = Collections.emptyList(); + + /** + * The owned root folders, listed once per sync. + * + *

Every attachment question used to re-list them, so an account with N attachments issued + * several times N folder listings per sync before moving a byte — which is what surfaced as + * Drive rate-limit errors on larger libraries. A backend instance lives for one sync; a root + * created concurrently by another device is the duplicate-root case the read path merges on the + * next sync anyway. + */ + @Nullable private List folderIds; + /** - * Blobs already read and hashed during this sync, keyed by root, hash and expected size. + * Blobs already verified during this sync, keyed by candidate and hash, with their size. * *

One attachment used to be downloaded in full two or three times per sync: once by * hasAttachment, once by the service re-verifying it, and once more while materializing it in * the canonical root. The verification itself is the point, so it still happens — once. */ - private final Set verifiedAttachments = new HashSet<>(); + private final Map verifiedAttachments = new HashMap<>(); public GoogleDriveSyncBackend(@NonNull String accessToken) { this(accessToken, DEFAULT_API, DEFAULT_UPLOAD, Clock.systemUTC(), new SyncBundleCodec()); @@ -78,6 +108,17 @@ public GoogleDriveSyncBackend(@NonNull String accessToken) { @NonNull String uploadBase, @NonNull Clock clock, @NonNull SyncBundleCodec bundleCodec) { + this(accessToken, apiBase, uploadBase, clock, bundleCodec, MAX_ATTACHMENT_RESPONSE_BYTES); + } + + /** Test seam: a small attachment ceiling makes the oversize paths reachable in a test. */ + GoogleDriveSyncBackend( + @NonNull String accessToken, + @NonNull String apiBase, + @NonNull String uploadBase, + @NonNull Clock clock, + @NonNull SyncBundleCodec bundleCodec, + long maxAttachmentBytes) { if (accessToken.trim().isEmpty()) { throw new IllegalArgumentException("accessToken is empty"); } @@ -86,6 +127,7 @@ public GoogleDriveSyncBackend(@NonNull String accessToken) { this.uploadBase = uploadBase; this.clock = clock; this.bundleCodec = bundleCodec; + this.maxAttachmentBytes = maxAttachmentBytes; this.requestExecutor = new DriveRequestExecutor(); } @@ -95,17 +137,21 @@ public String getIdentifier() { return "google-drive"; } + /** The merged remote records alone; a convenience for tests, not part of the protocol. */ @NonNull - @Override - public synchronized SyncSnapshot readSnapshot() throws IOException { + synchronized SyncSnapshot readSnapshot() throws IOException { return readSnapshotResult().getSnapshot(); } + @NonNull @Override public synchronized RemoteSnapshot readSnapshotResult() throws IOException { - List folderIds = findFolderIds(); - if (folderIds.isEmpty()) { - lastReadFrontierBundleIds = Collections.emptyList(); + // A read begins a sync, so it sees the roots as they are now; the listing then serves + // every attachment question until the next read. + folderIds = listFolderIds(); + List roots = folderIds; + if (roots.isEmpty()) { + lastReadBundles = Collections.emptyList(); lastReadToken = UUID.randomUUID().toString(); return new RemoteSnapshot( SyncSnapshot.empty(), @@ -117,19 +163,25 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { } Map bundlesByLogicalId = new HashMap<>(); - Map bytesByLogicalId = new HashMap<>(); - for (String folderId : folderIds) { - for (String bundleId : findBundles(folderId)) { + // Only a digest per logical bundle is retained for the duplicate-copy check; holding + // every bundle's bytes for the whole read grew with the account's history. + Map digestByLogicalId = new HashMap<>(); + List bundleFiles = new ArrayList<>(); + for (String folderId : roots) { + for (BundleFile file : findBundles(folderId)) { byte[] bytes = requestBytes( "GET", - apiBase + "/files/" + bundleId + "?alt=media", + apiBase + "/files/" + file.fileId + "?alt=media", MAX_BUNDLE_RESPONSE_BYTES); SyncBundleCodec.DecodedBundle decoded = bundleCodec.decode(new ByteArrayInputStream(bytes)); - byte[] previousBytes = bytesByLogicalId.putIfAbsent(decoded.getBundleId(), bytes); - if (previousBytes != null) { - if (!java.util.Arrays.equals(previousBytes, bytes)) { + String digest = Sha256.of(bytes); + bundleFiles.add(file.withLogicalId(decoded.getBundleId())); + String previousDigest = + digestByLogicalId.putIfAbsent(decoded.getBundleId(), digest); + if (previousDigest != null) { + if (!previousDigest.equals(digest)) { throw new IOException( "Drive contains conflicting physical copies of one bundle"); } @@ -187,24 +239,20 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { SyncMergeResult.Source.REMOTE)); } - lastReadFrontierBundleIds = Collections.unmodifiableList(new ArrayList<>(frontier)); + lastReadBundles = Collections.unmodifiableList(bundleFiles); lastReadToken = UUID.randomUUID().toString(); return new RemoteSnapshot( merged, conflicts, frontier, alternatives, resolvedAlternativeIds, lastReadToken); } - @Override - public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException { - throw new IOException( - "A Drive publish requires the read context it was derived from; use publish()"); - } - @Override public synchronized void publish(@NonNull SyncPublication publication) throws IOException { // Causal parents used to come from a mutable field, so a write with no preceding read // published a parentless root that permanently forked the DAG. The read that produced - // this publication has to be this backend's most recent one. - String token = publication.getReadContext().getReadToken(); + // this publication has to be this backend's most recent one, and it is that read — not a + // second copy of its frontier kept on this object — that names the parents. + RemoteSnapshot readContext = publication.getReadContext(); + String token = readContext.getReadToken(); if (token.isEmpty() || !token.equals(lastReadToken)) { throw new IOException( "Drive publish is not derived from this backend's latest remote read"); @@ -221,7 +269,7 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO bundleCodec.encode( snapshot, clock.instant(), - lastReadFrontierBundleIds, + readContext.getFrontierBundleIds(), publication.getUnresolvedAlternatives(), publication.getResolvedAlternativeIds()); // Every bundle is immutable. Drive offers no conditional update based on its version @@ -235,10 +283,40 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO // POST is deliberately not blindly retried. The server may have accepted the upload // before the client lost its response; rediscovering the unique name makes that // outcome successful without publishing a second logical bundle. - if (!hasBundleNamed(folderId, bundleName)) { + if (!mayHaveCommitted(uploadFailure) || !hasBundleNamed(folderId, bundleName)) { throw uploadFailure; } } + pruneSupersededBundles(readContext.getFrontierBundleIds()); + } + + /** + * Retires the bundles the one just published makes redundant. + * + *

Nothing ever deleted a bundle, so an account accumulated one full snapshot per changed + * sync, and every later sync downloaded, unzipped and validated all of them to compute a + * frontier of one or two heads. A bundle is a complete snapshot, so everything a superseded + * bundle held — records, tombstones, unresolved alternatives — lives on in its descendants, and + * the read path already tolerates a missing ancestor. The heads this publish descended from are + * kept for now: they are what a concurrent publisher is about to name as parents. They go at + * the next sync, once the grace period has passed. + * + *

Best effort by design: a bundle that cannot be removed costs a download next time, never + * correctness. + */ + private void pruneSupersededBundles(@NonNull Collection frontierBundleIds) { + long cutoff = clock.millis() - BUNDLE_PRUNE_GRACE_MILLIS; + for (BundleFile bundle : lastReadBundles) { + if (frontierBundleIds.contains(bundle.logicalId) + || (bundle.publishedAtMillis != null && bundle.publishedAtMillis > cutoff)) { + continue; + } + try { + deleteFile(bundle.fileId); + } catch (IOException ignored) { + // Still there next time; the read path copes with it either way. + } + } } /** @@ -251,36 +329,46 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO @Override public synchronized boolean hasAttachment(@NonNull String sha256) throws IOException { for (String folderId : findFolderIds()) { - if (findAttachment(folderId, sha256) != null) { + if (!listAttachmentCandidates(folderId, sha256).isEmpty()) { return true; } } return false; } + /** + * Opens one copy of the blob, choosing a verified candidate where one is known. + * + *

The stream is not verified here: every caller wraps it in its own verifier, and doing it + * here as well meant downloading the blob twice — once to check it and once to hand it over. + * Only when several unverified copies exist is one read ahead of time, so that a corrupt + * duplicate cannot be the one handed to the caller. + */ @Nullable @Override public synchronized InputStream readAttachment(@NonNull String sha256) throws IOException { for (String folderId : findFolderIds()) { - String attachmentId = findVerifiedAttachment(folderId, sha256, null); - if (attachmentId == null) { + List candidates = listAttachmentCandidates(folderId, sha256); + if (candidates.isEmpty()) { continue; } - // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing - // ByteArrayOutputStream first doubled, then copied) was the largest single allocation - // in - // the sync and an OutOfMemoryError on an ordinary phone. - HttpURLConnection connection = - requestExecutor.executeIdempotent( - () -> - openSuccessful( - "GET", - apiBase + "/files/" + attachmentId + "?alt=media")); - try { - return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); - } catch (IOException failure) { - connection.disconnect(); - throw failure; + String chosen = null; + for (AttachmentCandidate candidate : candidates) { + if (isVerifiedWithoutReading(candidate, sha256, null)) { + chosen = candidate.id; + break; + } + } + if (chosen == null && candidates.size() == 1) { + // A single corrupt copy fails the caller's verifier exactly as it would fail + // one here; the difference is one download instead of two. + chosen = candidates.get(0).id; + } + if (chosen == null) { + chosen = findVerifiedAttachment(folderId, sha256, null); + } + if (chosen != null) { + return openAttachment(chosen); } } return null; @@ -294,67 +382,77 @@ public synchronized void writeAttachment( if (findVerifiedAttachment(folderId, sha256, sizeBytes >= 0L ? sizeBytes : null) != null) { return; } - if (sizeBytes >= 0L) { - if (sizeBytes > MAX_ATTACHMENT_RESPONSE_BYTES) { - throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); - } - uploadAttachmentOrConfirm(folderId, sha256, content, sizeBytes); + if (sizeBytes < 0L) { + // No declared size, so the content length cannot be computed up front. Rare: sizes + // come from the bundle manifest, which also supplies the hashes being uploaded. + byte[] buffered = + readBounded( + content, + maxAttachmentBytes, + "Attachment exceeds the 100 MiB sync upload limit"); + uploadAttachmentOrConfirm( + folderId, sha256, new ByteArrayInputStream(buffered), buffered.length); return; } - // No declared size, so the multipart content length cannot be computed up front. Rare: - // sizes come from the bundle manifest, which also supplies the hashes being uploaded. - uploadAttachmentOrConfirm( - folderId, sha256, readFullyLimited(content, MAX_ATTACHMENT_RESPONSE_BYTES)); + if (sizeBytes > maxAttachmentBytes) { + throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); + } + uploadAttachmentOrConfirm(folderId, sha256, content, sizeBytes); } + @NonNull private List findFolderIds() throws IOException { + if (folderIds == null) { + folderIds = listFolderIds(); + } + return folderIds; + } + + @NonNull + private List listFolderIds() throws IOException { JsonArray folders = listFiles( "mimeType = '" + MIME_FOLDER + "' and trashed = false and " - + appPropertyClause("mynotesOwner", "1"), + + appPropertyClause(PROPERTY_OWNER, "1"), "files(id,name)"); List result = new ArrayList<>(folders.size()); for (int index = 0; index < folders.size(); index++) { result.add(folders.get(index).getAsJsonObject().get("id").getAsString()); } result.sort(Comparator.naturalOrder()); - return result; - } - - /** Deterministically selects one byte-identical content-addressed attachment duplicate. */ - @Nullable - private static String smallestId(@NonNull JsonArray files) { - String selected = null; - for (int index = 0; index < files.size(); index++) { - String id = files.get(index).getAsJsonObject().get("id").getAsString(); - if (selected == null || id.compareTo(selected) < 0) { - selected = id; - } - } - return selected; + return Collections.unmodifiableList(result); } @NonNull private String ensureCanonicalFolderId() throws IOException { - List folderIds = findFolderIds(); - if (!folderIds.isEmpty()) { - return folderIds.get(0); + List roots = findFolderIds(); + if (roots.isEmpty()) { + // The cached answer may predate another device's first sync; only a fresh listing + // may justify creating a root. + roots = listFolderIds(); + folderIds = roots; + } + if (!roots.isEmpty()) { + return roots.get(0); } JsonObject metadata = new JsonObject(); metadata.addProperty("name", FOLDER_NAME); metadata.addProperty("mimeType", MIME_FOLDER); - metadata.add("appProperties", appProperties("mynotesOwner", "1")); + metadata.add("appProperties", appProperties(PROPERTY_OWNER, "1")); try { - return uploadMetadata(metadata); + String created = uploadMetadata(metadata); + folderIds = Collections.singletonList(created); + return created; } catch (IOException createFailure) { // Folder POST can have committed before a lost response. Duplicate roots are a // supported read state; rediscovery avoids a blind retry creating another one. - folderIds = findFolderIds(); - if (!folderIds.isEmpty()) { - return folderIds.get(0); + roots = listFolderIds(); + folderIds = roots; + if (!roots.isEmpty()) { + return roots.get(0); } throw createFailure; } @@ -380,8 +478,8 @@ private void materializeAttachmentsInCanonicalRoot( if (source == null) { throw new IOException("Required attachment is unavailable in any Drive root"); } - try (VerifiedAttachmentInputStream input = - new VerifiedAttachmentInputStream(source, hash, attachment.getValue())) { + try (VerifyingInputStream input = + new VerifyingInputStream(source, hash, attachment.getValue())) { uploadAttachmentOrConfirm(canonicalRootId, hash, input, attachment.getValue()); input.verifyEndOfStream(); } @@ -411,7 +509,7 @@ private void ensureCanonicalAlternativeAttachments( } @NonNull - private static Map attachmentSizes(@NonNull Collection notes) + private Map attachmentSizes(@NonNull Collection notes) throws IOException { Map sizes = new HashMap<>(); for (SyncRecord record : notes) { @@ -426,7 +524,7 @@ private static Map attachmentSizes(@NonNull Collection } String hash = entry.get("sha256").getAsString(); long size = entry.get("size").getAsLong(); - if (size < 0L || size > MAX_ATTACHMENT_RESPONSE_BYTES) { + if (size < 0L || size > maxAttachmentBytes) { throw new IOException("Attachment size exceeds the sync limit"); } Long previous = sizes.putIfAbsent(hash, size); @@ -451,32 +549,62 @@ private static Map attachmentSizes(@NonNull Collection */ private static void validateBundleDag( @NonNull Map bundles) throws IOException { - Set visiting = new HashSet<>(); - Set visited = new HashSet<>(); - for (String bundleId : bundles.keySet()) { - validateAcyclic(bundleId, bundles, visiting, visited); + Map> parentsById = new HashMap<>(); + for (Map.Entry entry : bundles.entrySet()) { + parentsById.put(entry.getKey(), entry.getValue().getParentBundleIds()); } + validateAncestry(parentsById); } - private static void validateAcyclic( - @NonNull String bundleId, - @NonNull Map bundles, - @NonNull Set visiting, - @NonNull Set visited) + /** + * Rejects a cycle in the parent graph. + * + *

Iterative on purpose: the recursive walk went one frame deeper per ancestor, so a long + * linear history — exactly what an account that syncs after every edit accumulates — could + * overflow the worker's stack, and a {@code StackOverflowError} is not an {@code IOException} + * the sync knows how to report. + */ + static void validateAncestry(@NonNull Map> parentsById) throws IOException { - if (visited.contains(bundleId)) return; - SyncBundleCodec.DecodedBundle bundle = bundles.get(bundleId); - if (bundle == null) { - // An ancestor that is no longer stored. Nothing to walk and nothing to lose. - return; + Set visited = new HashSet<>(); + Set visiting = new HashSet<>(); + Deque stack = new ArrayDeque<>(); + for (String root : parentsById.keySet()) { + if (visited.contains(root)) { + continue; + } + visiting.add(root); + stack.push(new Frame(root, parentsById.get(root).iterator())); + while (!stack.isEmpty()) { + Frame frame = stack.peek(); + if (!frame.parents.hasNext()) { + stack.pop(); + visiting.remove(frame.bundleId); + visited.add(frame.bundleId); + continue; + } + String parent = frame.parents.next(); + Collection grandparents = parentsById.get(parent); + if (grandparents == null || visited.contains(parent)) { + // An ancestor that is no longer stored, or one already walked. + continue; + } + if (!visiting.add(parent)) { + throw new IOException("Drive bundle ancestry contains a cycle"); + } + stack.push(new Frame(parent, grandparents.iterator())); + } } - if (!visiting.add(bundleId)) - throw new IOException("Drive bundle ancestry contains a cycle"); - for (String parent : bundle.getParentBundleIds()) { - validateAcyclic(parent, bundles, visiting, visited); + } + + private static final class Frame { + private final String bundleId; + private final Iterator parents; + + private Frame(String bundleId, Iterator parents) { + this.bundleId = bundleId; + this.parents = parents; } - visiting.remove(bundleId); - visited.add(bundleId); } @NonNull @@ -494,68 +622,121 @@ private static List computeFrontier( return frontier; } + /** One physical bundle file in a root; the logical id is known once it has been decoded. */ + private static final class BundleFile { + private final String fileId; + @Nullable private final String logicalId; + @Nullable private final Long publishedAtMillis; + + private BundleFile( + @NonNull String fileId, + @Nullable String logicalId, + @Nullable Long publishedAtMillis) { + this.fileId = fileId; + this.logicalId = logicalId; + this.publishedAtMillis = publishedAtMillis; + } + + @NonNull + private BundleFile withLogicalId(@NonNull String id) { + return new BundleFile(fileId, id, publishedAtMillis); + } + } + @NonNull - private List findBundles(@NonNull String folderId) throws IOException { + private List findBundles(@NonNull String folderId) throws IOException { JsonArray bundles = listFiles( - "'" - + folderId - + "' in parents and trashed = false and " - + appPropertyClause("mynotesBundle", "1"), - "files(id,name)"); - List result = new ArrayList<>(bundles.size()); + ownedFilesQuery(folderId, PROPERTY_BUNDLE, "1"), + "files(id,name,appProperties)"); + List result = new ArrayList<>(bundles.size()); for (int index = 0; index < bundles.size(); index++) { - result.add(bundles.get(index).getAsJsonObject().get("id").getAsString()); - } - result.sort(Comparator.naturalOrder()); + JsonObject file = bundles.get(index).getAsJsonObject(); + result.add( + new BundleFile( + file.get("id").getAsString(), + null, + publishedAtOf(file.getAsJsonObject("appProperties")))); + } + result.sort(Comparator.comparing(file -> file.fileId)); return result; } @Nullable - private String findAttachment(@NonNull String folderId, @NonNull String sha256) - throws IOException { + private static Long publishedAtOf(@Nullable JsonObject appProperties) { + if (appProperties == null || !appProperties.has(PROPERTY_BUNDLE_PUBLISHED_AT)) { + return null; + } + try { + return Long.parseLong(appProperties.get(PROPERTY_BUNDLE_PUBLISHED_AT).getAsString()); + } catch (RuntimeException malformed) { + return null; + } + } + + /** One Drive object indexed under a hash, with what Drive itself says about its bytes. */ + private static final class AttachmentCandidate { + private final String id; + @Nullable private final Long size; + @Nullable private final String sha256Checksum; + + private AttachmentCandidate( + @NonNull String id, @Nullable Long size, @Nullable String sha256Checksum) { + this.id = id; + this.size = size; + this.sha256Checksum = sha256Checksum; + } + } + + /** Every object in the root indexed under {@code sha256}, smallest id first. */ + @NonNull + private List listAttachmentCandidates( + @NonNull String folderId, @NonNull String sha256) throws IOException { JsonArray files = listFiles( - "'" - + folderId - + "' in parents and trashed = false and " - + appPropertyClause("mynotesAttachmentSha256", sha256), - "files(id,name)"); + ownedFilesQuery(folderId, PROPERTY_ATTACHMENT_SHA256, sha256), + "files(id,name,size,sha256Checksum)"); + List candidates = new ArrayList<>(files.size()); + for (int index = 0; index < files.size(); index++) { + JsonObject file = files.get(index).getAsJsonObject(); + candidates.add( + new AttachmentCandidate( + file.get("id").getAsString(), + optionalLong(file, "size"), + optionalString(file, "sha256Checksum"))); + } // Attachments are content-addressed, so duplicates uploaded by two devices racing on the // same hash are byte-identical and either one will do. Rejecting them used to break every // subsequent sync permanently. - return smallestId(files); + candidates.sort(Comparator.comparing(candidate -> candidate.id)); + return candidates; } /** - * An app property is only an index. Read and verify every candidate before it may satisfy a + * An app property is only an index. Every candidate is verified before it may satisfy a * content-addressed reference; corrupt candidates remain harmless Drive orphans. + * + *

Drive computes a checksum over the bytes it stores and reports it with the listing, so a + * candidate is usually verified without a download; only an object Drive has not checksummed is + * read. The old version read every candidate in full on every sync, which for a library of a + * few hundred megabytes meant re-downloading all of it on every six-hourly run. */ @Nullable private String findVerifiedAttachment( @NonNull String folderId, @NonNull String sha256, @Nullable Long expectedSize) throws IOException { - JsonArray files = - listFiles( - "'" - + folderId - + "' in parents and trashed = false and " - + appPropertyClause("mynotesAttachmentSha256", sha256), - "files(id,name)"); - List candidateIds = new ArrayList<>(files.size()); - for (int index = 0; index < files.size(); index++) { - candidateIds.add(files.get(index).getAsJsonObject().get("id").getAsString()); - } - candidateIds.sort(Comparator.naturalOrder()); - for (String candidateId : candidateIds) { - String cacheKey = candidateId + "\u0000" + sha256 + "\u0000" + expectedSize; - if (verifiedAttachments.contains(cacheKey)) { - return candidateId; + for (AttachmentCandidate candidate : listAttachmentCandidates(folderId, sha256)) { + if (isVerifiedWithoutReading(candidate, sha256, expectedSize)) { + return candidate.id; } - try (InputStream candidate = openAttachment(candidateId)) { - verifyAttachment(candidate, sha256, expectedSize); - verifiedAttachments.add(cacheKey); - return candidateId; + if (candidate.sha256Checksum != null) { + // Drive's own digest disagrees with the index; reading would only confirm it. + continue; + } + try (InputStream content = openAttachment(candidate.id)) { + long size = VerifyingInputStream.verify(content, sha256, expectedSize); + verifiedAttachments.put(memoKey(candidate.id, sha256), size); + return candidate.id; } catch (AttachmentIntegrityException corrupt) { // A second content-addressed duplicate may be valid. Never accept the property // alone and never delete this object during a correctness path. @@ -564,10 +745,31 @@ private String findVerifiedAttachment( return null; } + /** True when this sync, or Drive's own checksum, already vouches for the candidate's bytes. */ + private boolean isVerifiedWithoutReading( + @NonNull AttachmentCandidate candidate, + @NonNull String sha256, + @Nullable Long expectedSize) { + Long verifiedSize = verifiedAttachments.get(memoKey(candidate.id, sha256)); + if (verifiedSize == null + && sha256.equals(candidate.sha256Checksum) + && candidate.size != null + && candidate.size <= maxAttachmentBytes) { + verifiedSize = candidate.size; + verifiedAttachments.put(memoKey(candidate.id, sha256), verifiedSize); + } + return verifiedSize != null && (expectedSize == null || expectedSize.equals(verifiedSize)); + } + + @NonNull + private static String memoKey(@NonNull String candidateId, @NonNull String sha256) { + return candidateId + "" + sha256; + } + /** * True only when a remote blob exists and its actual bytes hash to {@code sha256}. * - *

Drive's {@code appProperties} index is a claim, not proof, so the bytes are read. The + *

Drive's {@code appProperties} index is a claim, not proof, so the bytes are checked. The * result is remembered for this sync so the caller does not have to download the blob again * purely to repeat the same check. */ @@ -584,58 +786,22 @@ public synchronized boolean hasVerifiedAttachment( @NonNull private InputStream openAttachment(@NonNull String attachmentId) throws IOException { + // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing + // ByteArrayOutputStream first doubled, then copied) was the largest single allocation in + // the sync and an OutOfMemoryError on an ordinary phone. HttpURLConnection connection = requestExecutor.executeIdempotent( () -> openSuccessful( "GET", apiBase + "/files/" + attachmentId + "?alt=media")); try { - return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); + return new ConnectionInputStream(connection, maxAttachmentBytes); } catch (IOException failure) { connection.disconnect(); throw failure; } } - private static void verifyAttachment( - @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) - throws IOException { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - long size = 0L; - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - digest.update(buffer, 0, read); - size += read; - if (size > MAX_ATTACHMENT_RESPONSE_BYTES) { - throw new AttachmentIntegrityException("Attachment exceeds the sync size limit"); - } - } - String actual = toHex(digest.digest()); - if (!expectedHash.equals(actual)) { - throw new AttachmentIntegrityException( - "Attachment checksum does not match its declared hash"); - } - if (expectedSize != null && expectedSize.longValue() != size) { - throw new AttachmentIntegrityException( - "Attachment size does not match its declared size"); - } - } - - @NonNull - private static String toHex(@NonNull byte[] bytes) { - StringBuilder value = new StringBuilder(bytes.length * 2); - for (byte byteValue : bytes) { - value.append(String.format(Locale.US, "%02x", byteValue & 0xff)); - } - return value.toString(); - } - @NonNull private JsonArray listFiles(@NonNull String query, @NonNull String fields) throws IOException { JsonArray result = new JsonArray(); @@ -677,6 +843,20 @@ private String uploadMetadata(@NonNull JsonObject metadata) throws IOException { return created.get("id").getAsString(); } + /** Permanently removes one file this app created; bundles are complete, so nothing is lost. */ + private void deleteFile(@NonNull String fileId) throws IOException { + HttpURLConnection connection = open("DELETE", apiBase + "/files/" + fileId); + try { + int code = connection.getResponseCode(); + if (code == HttpURLConnection.HTTP_NOT_FOUND) { + return; + } + ensureSuccess(connection); + } finally { + connection.disconnect(); + } + } + private boolean hasBundleNamed(@NonNull String folderId, @NonNull String name) throws IOException { JsonArray bundles = @@ -686,11 +866,21 @@ private boolean hasBundleNamed(@NonNull String folderId, @NonNull String name) + "' in parents and trashed = false and name = '" + escapeQuery(name) + "' and " - + appPropertyClause("mynotesBundle", "1"), + + appPropertyClause(PROPERTY_BUNDLE, "1"), "files(id)"); return bundles.size() > 0; } + /** The one spelling of "our file, in this root, indexed under this property". */ + @NonNull + private static String ownedFilesQuery( + @NonNull String folderId, @NonNull String key, @NonNull String value) { + return "'" + + folderId + + "' in parents and trashed = false and " + + appPropertyClause(key, value); + } + private void uploadFile( @NonNull String folderId, @NonNull String name, @@ -729,7 +919,7 @@ private void uploadEmptyAttachment( @NonNull InputStream content) throws IOException { if (content.read() != -1) { - throw new IOException("Attachment exceeds its declared size"); + throw new AttachmentIntegrityException("Attachment exceeds its declared size"); } uploadMultipart(folderId, name, mimeType, new ByteArrayInputStream(new byte[0]), 0L, false); } @@ -744,6 +934,12 @@ private void uploadEmptyAttachment( * never terminated, and it replayed the buffer under offsets past the end of the file. Nothing * here is derived — the buffer window is recomputed from absolute offsets on every pass, so a * byte can only ever be sent under the one offset it occupies in the source. + * + *

Every request but the last carries a whole 256 KiB window, which is what Drive's resumable + * protocol requires. After a partial acknowledgement the window therefore slides: the + * unacknowledged tail moves to the front of the buffer and the window is refilled from the + * source. Sending only the tail — a short chunk that is not the last — was answered with a 400 + * that nothing retries. */ private void uploadResumableAttachment( @NonNull String folderId, @@ -762,31 +958,36 @@ private void uploadResumableAttachment( while (acknowledgedExclusive < sizeBytes) { throwIfInterrupted(); - if (acknowledgedExclusive >= bufferStart + bufferLength) { - // Everything buffered is durable; read the next window from the source. + if (acknowledgedExclusive > bufferStart) { + // Everything before the acknowledged offset is durable; keep only the tail. + int consumed = (int) (acknowledgedExclusive - bufferStart); + if (consumed >= bufferLength) { + bufferLength = 0; + } else { + System.arraycopy(buffer, consumed, buffer, 0, bufferLength - consumed); + bufferLength -= consumed; + } bufferStart = acknowledgedExclusive; - bufferLength = - readChunk( - content, - buffer, - (int) Math.min(buffer.length, sizeBytes - bufferStart)); - if (bufferLength <= 0) { - throw new IOException("Attachment ended before its declared size"); + } + int wanted = (int) Math.min(buffer.length, sizeBytes - bufferStart); + if (bufferLength < wanted) { + bufferLength += readChunk(content, buffer, bufferLength, wanted - bufferLength); + if (bufferLength < wanted) { + // The source disagrees with its own manifest: an integrity failure, so it + // is never mistaken for a lost response worth confirming by discovery. + throw new AttachmentIntegrityException( + "Attachment ended before its declared size"); } } - int offsetInBuffer = (int) (acknowledgedExclusive - bufferStart); - int length = bufferLength - offsetInBuffer; - long chunkStart = acknowledgedExclusive; + int length = bufferLength; + long chunkStart = bufferStart; long chunkEndExclusive = chunkStart + length; // A chunk PUT is idempotent: it is addressed by an absolute Content-Range, so a // replay of the identical range either lands at the same offset or is already // committed. Retrying is therefore safe, and it keeps one transient 5xx between // chunks from discarding a large upload that is nearly complete. - final int retryOffset = offsetInBuffer; - final int retryLength = length; - final long retryStart = chunkStart; long reported = requestExecutor.executeIdempotent( () -> @@ -794,9 +995,9 @@ private void uploadResumableAttachment( sessionUrl, mimeType, buffer, - retryOffset, - retryLength, - retryStart, + 0, + length, + chunkStart, sizeBytes)); if (reported < acknowledgedExclusive) { @@ -822,7 +1023,7 @@ private void uploadResumableAttachment( } if (content.read() != -1) { - throw new IOException("Attachment exceeds its declared size"); + throw new AttachmentIntegrityException("Attachment exceeds its declared size"); } } @@ -925,17 +1126,21 @@ private static long resumableAcknowledgedExclusive(@Nullable String range) throw } } - private static int readChunk(@NonNull InputStream input, @NonNull byte[] buffer, int maximum) + /** + * Fills {@code buffer} from {@code offset} with up to {@code count} bytes, or to end of input. + */ + private static int readChunk( + @NonNull InputStream input, @NonNull byte[] buffer, int offset, int count) throws IOException { - int offset = 0; - while (offset < maximum) { - int read = input.read(buffer, offset, maximum - offset); + int filled = 0; + while (filled < count) { + int read = input.read(buffer, offset + filled, count - filled); if (read == -1) { break; } - offset += read; + filled += read; } - return offset; + return filled; } private static void throwIfInterrupted() throws IOException { @@ -954,7 +1159,7 @@ private static JsonObject attachmentMetadata(@NonNull String folderId, @NonNull parents.add(folderId); metadata.add("parents", parents); JsonObject properties = new JsonObject(); - properties.addProperty("mynotesAttachmentSha256", sha256); + properties.addProperty(PROPERTY_ATTACHMENT_SHA256, sha256); metadata.add("appProperties", properties); return metadata; } @@ -965,47 +1170,53 @@ private void uploadAttachmentOrConfirm( @NonNull InputStream content, long sizeBytes) throws IOException { + if (sizeBytes < 0L) { + throw new IllegalArgumentException("An attachment upload needs a declared size"); + } try { - if (sizeBytes >= 0L) { - uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); - } else { - uploadFile(folderId, sha256, MIME_BINARY, readFully(content), false); - } + uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); } catch (IOException uploadFailure) { // Attachment identity is its SHA-256. A successful request whose response was lost is // confirmed by discovery, not repeated with an already-consumed stream. - if (!isAmbiguousTransportFailure(uploadFailure) + if (!mayHaveCommitted(uploadFailure) || findVerifiedAttachment(folderId, sha256, sizeBytes) == null) { throw uploadFailure; } } } - private void uploadAttachmentOrConfirm( - @NonNull String folderId, @NonNull String sha256, @NonNull byte[] content) - throws IOException { - try { - uploadFile(folderId, sha256, MIME_BINARY, content, false); - } catch (IOException uploadFailure) { - if (!isAmbiguousTransportFailure(uploadFailure) - || findVerifiedAttachment(folderId, sha256, (long) content.length) == null) { - throw uploadFailure; - } + /** + * Whether a failed create may nonetheless have been committed by Drive. + * + *

The one answer for bundle and attachment uploads alike; they used to classify differently, + * so a 429 on the bundle POST was followed by rediscovery while the same 429 on an attachment + * POST failed the sync outright. A response Drive definitely never acted on — a rejected + * request, a bad token, a blob that failed its own checksum — is a plain failure. Everything + * else, including a read timeout that arrived after the whole body was sent, is worth one + * listing to find out. + */ + static boolean mayHaveCommitted(@NonNull IOException failure) { + if (failure instanceof AttachmentIntegrityException) { + return false; } - } - - private static boolean isAmbiguousTransportFailure(@NonNull IOException failure) { - if (failure instanceof AttachmentIntegrityException - || failure instanceof java.io.InterruptedIOException) { + // A subclass of InterruptedIOException, so it has to be asked about first: a timeout + // waiting for the response is precisely the case where the upload may have landed. + if (failure instanceof java.net.SocketTimeoutException) { + return true; + } + if (failure instanceof java.io.InterruptedIOException) { return false; } if (failure instanceof DriveRequestExecutor.DriveHttpException) { - int status = ((DriveRequestExecutor.DriveHttpException) failure).statusCode; - return status >= 500 && status <= 599; + DriveRequestExecutor.DriveHttpException http = + (DriveRequestExecutor.DriveHttpException) failure; + int status = http.statusCode; + if (status >= 500 || status == 408 || status == 429) { + return true; + } + return status == 403 && http.isRateLimit(); } - return failure instanceof java.net.SocketException - || failure instanceof java.net.SocketTimeoutException - || failure instanceof java.net.ConnectException; + return true; } /** @@ -1033,10 +1244,10 @@ private void uploadMultipart( JsonObject appProperties = new JsonObject(); if (bundleFile) { - appProperties.addProperty("mynotesBundle", "1"); - appProperties.addProperty("mynotesBundlePublishedAt", Long.toString(clock.millis())); + appProperties.addProperty(PROPERTY_BUNDLE, "1"); + appProperties.addProperty(PROPERTY_BUNDLE_PUBLISHED_AT, Long.toString(clock.millis())); } else { - appProperties.addProperty("mynotesAttachmentSha256", name); + appProperties.addProperty(PROPERTY_ATTACHMENT_SHA256, name); } metadata.add("appProperties", appProperties); @@ -1137,7 +1348,13 @@ private JsonObject readJsonResponse(@NonNull HttpURLConnection connection) throw ensureSuccess(connection); try (InputStream input = connection.getInputStream()) { return GSON.fromJson( - new String(readFully(input), StandardCharsets.UTF_8), JsonObject.class); + new String( + readBounded( + input, + MAX_BUNDLE_RESPONSE_BYTES, + "Drive response exceeds the sync size limit"), + StandardCharsets.UTF_8), + JsonObject.class); } } @@ -1154,16 +1371,7 @@ private byte[] requestBytesOnce(@NonNull String method, @NonNull String url, int try { ensureSuccess(connection); try (InputStream input = connection.getInputStream()) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - if (output.size() > maxBytes - read) { - throw new IOException("Drive response exceeds the sync size limit"); - } - output.write(buffer, 0, read); - } - return output.toByteArray(); + return readBounded(input, maxBytes, "Drive response exceeds the sync size limit"); } } finally { connection.disconnect(); @@ -1237,34 +1445,21 @@ private static String readErrorDetail(@Nullable InputStream error) { } } + /** Reads to end of stream, refusing anything past {@code maxBytes}; never closes the input. */ @NonNull - private static byte[] readFully(@NonNull InputStream input) throws IOException { - try (InputStream stream = input; - ByteArrayOutputStream output = new ByteArrayOutputStream()) { - byte[] buffer = new byte[8192]; - int read; - while ((read = stream.read(buffer)) != -1) { - output.write(buffer, 0, read); - } - return output.toByteArray(); - } - } - - @NonNull - private static byte[] readFullyLimited(@NonNull InputStream input, int maxBytes) + private static byte[] readBounded( + @NonNull InputStream input, long maxBytes, @NonNull String limitMessage) throws IOException { - try (InputStream stream = input; - ByteArrayOutputStream output = new ByteArrayOutputStream()) { - byte[] buffer = new byte[8192]; - int read; - while ((read = stream.read(buffer)) != -1) { - if (output.size() > maxBytes - read) { - throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); - } - output.write(buffer, 0, read); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + if (output.size() > maxBytes - read) { + throw new IOException(limitMessage); } - return output.toByteArray(); + output.write(buffer, 0, read); } + return output.toByteArray(); } @NonNull @@ -1293,11 +1488,34 @@ private static String escapeQuery(@NonNull String value) { return value.replace("\\", "\\\\").replace("'", "\\'"); } + @Nullable + private static String optionalString(@NonNull JsonObject object, @NonNull String field) { + JsonElement value = object.get(field); + return value == null || value.isJsonNull() || !value.isJsonPrimitive() + ? null + : value.getAsString(); + } + + @Nullable + private static Long optionalLong(@NonNull JsonObject object, @NonNull String field) { + String value = optionalString(object, field); + if (value == null) { + return null; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException malformed) { + return null; + } + } + /** * A response body that stays attached to its connection until the reader is done. * *

Lets an attachment be piped straight from the socket to disk while still enforcing the - * response ceiling, and releases the connection on close. + * response ceiling, and releases the connection on close. Exceeding the ceiling is an integrity + * failure like any other: a candidate that grows past it is skipped in favour of the next copy + * rather than failing the sync, as a plain I/O error used to. */ private static final class ConnectionInputStream extends FilterInputStream { private final HttpURLConnection connection; @@ -1332,7 +1550,7 @@ public int read(byte[] buffer, int offset, int length) throws IOException { private void count(int read) throws IOException { byteCount += read; if (byteCount > maxBytes) { - throw new IOException("Drive response exceeds the sync size limit"); + throw new AttachmentIntegrityException("Attachment exceeds the sync size limit"); } } @@ -1345,68 +1563,4 @@ public void close() throws IOException { } } } - - /** Verifies an untrusted remote blob before it may support canonical bundle publication. */ - private static final class VerifiedAttachmentInputStream extends FilterInputStream { - private final String expectedHash; - private final long expectedSize; - private final MessageDigest digest; - private long size; - private boolean reachedEnd; - - private VerifiedAttachmentInputStream( - @NonNull InputStream source, @NonNull String expectedHash, long expectedSize) - throws IOException { - super(source); - this.expectedHash = expectedHash; - this.expectedSize = expectedSize; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - } - - @Override - public int read() throws IOException { - int value = super.read(); - if (value == -1) { - reachedEnd = true; - } else { - digest.update((byte) value); - size++; - } - return value; - } - - @Override - public int read(byte[] buffer, int offset, int length) throws IOException { - int read = super.read(buffer, offset, length); - if (read == -1) { - reachedEnd = true; - } else if (read > 0) { - digest.update(buffer, offset, read); - size += read; - } - return read; - } - - private void verifyEndOfStream() throws IOException { - if (!reachedEnd) { - throw new IOException("Attachment upload ended before the source was verified"); - } - if (size != expectedSize) { - throw new AttachmentIntegrityException( - "Attachment size does not match sync metadata"); - } - StringBuilder actualHash = new StringBuilder(64); - for (byte value : digest.digest()) { - actualHash.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); - } - if (!expectedHash.equals(actualHash.toString())) { - throw new AttachmentIntegrityException( - "Attachment checksum does not match sync metadata"); - } - } - } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java b/app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java new file mode 100644 index 00000000..1b04355d --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java @@ -0,0 +1,60 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.util.regex.Pattern; + +/** + * Decides whether the live preferences differ from the last value sync recorded. + * + *

SharedPreferences has no mutation hook, so a local settings change is only ever noticed by + * comparing a digest of the live values against the baseline stored at the last sync. The baseline + * format changed once: 2.6.49 stored the payload's raw JSON, later releases store its SHA-256. A + * device upgrading across that change still holds a JSON baseline, and comparing it against a + * digest can only ever say "changed" — which manufactured a local edit on every upgraded device and + * let its untouched settings outrank a genuine change made elsewhere. + * + *

Deliberately free of {@code android.*} so every branch runs under a plain JVM test. + */ +final class PreferencesBaselineDecision { + + enum Action { + /** Nothing changed since the last sync. */ + UNCHANGED, + /** + * The baseline predates the digest format but describes the live values; store the digest. + */ + MIGRATE_BASELINE, + /** The user changed a setting since the last sync; the record must be touched. */ + LOCAL_EDIT + } + + private static final Pattern DIGEST = Pattern.compile("[0-9a-f]{64}"); + + private PreferencesBaselineDecision() {} + + /** + * @param storedBaseline the baseline read from preferences, or {@code null} when none exists. + * @param liveDigest SHA-256 of the live preferences in the current format. + * @param legacyFingerprint the live preferences serialized the way 2.6.49 stored its baseline. + */ + @NonNull + static Action decide( + @Nullable String storedBaseline, + @NonNull String liveDigest, + @NonNull String legacyFingerprint) { + if (storedBaseline == null) { + // No baseline at all means sync has never seen these settings; treating them as a + // local edit is the conservative reading, because the alternative silently loses a + // fresh install's configuration to an older version already on Drive. + return Action.LOCAL_EDIT; + } + if (liveDigest.equals(storedBaseline)) { + return Action.UNCHANGED; + } + if (!DIGEST.matcher(storedBaseline).matches() && storedBaseline.equals(legacyFingerprint)) { + return Action.MIGRATE_BASELINE; + } + return Action.LOCAL_EDIT; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 0cd191a7..6f113cb1 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -21,6 +21,8 @@ import com.pasich.mynotes.data.model.TaskCategory; import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.extendedEditor.attach.AttachmentStorage; +import com.pasich.mynotes.extendedEditor.attach.AttachmentUrl; +import com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks; import com.pasich.mynotes.extendedEditor.models.EditorAttachment; import com.pasich.mynotes.utils.backup.models.PreferencesBackup; import java.io.File; @@ -31,14 +33,15 @@ import java.io.OutputStream; import java.net.URLConnection; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -49,6 +52,9 @@ public final class RoomSyncStore implements SyncStore { private static final String LEGACY_STATE = "last_state"; private static final String PREFERENCES_HASH = "preferences_hash"; private static final String PREFERENCES_STABLE_ID = "00000000-0000-4000-8000-000000000000"; + private static final String ATTACHMENT_CACHE_DIR = "sync-attachments"; + private static final String HASH_CACHE_FILE = "hash-cache.json"; + private static final String SHA_256 = "[0-9a-f]{64}"; private final AppDatabase database; private final SharedPreferences preferences; private volatile boolean seeded; @@ -58,6 +64,7 @@ public final class RoomSyncStore implements SyncStore { private final AttachmentResolver attachmentResolver; private final AttachmentHasher attachmentHasher; private final TransactionFailureInjector transactionFailureInjector; + private final AttachmentHashCache hashCache; /** * Content hash to the note-folder file holding it, indexed while the snapshot is built so the @@ -65,6 +72,9 @@ public final class RoomSyncStore implements SyncStore { */ private final Map localAttachments = new ConcurrentHashMap<>(); + /** Whether every note folder has been indexed into {@link #localAttachments}. */ + private volatile boolean noteFoldersIndexed; + /** * Set when an apply actually changed the visible settings, so the screen can redraw. * @@ -84,7 +94,7 @@ public RoomSyncStore( database, preferenceHelper, AttachmentStorage::resolve, - RoomSyncStore::sha256, + Sha256::of, record -> {}); } @@ -122,6 +132,11 @@ public RoomSyncStore( this.transactionFailureInjector = transactionFailureInjector; this.preferences = context.getApplicationContext().getSharedPreferences(PREFS, Context.MODE_PRIVATE); + this.hashCache = + new AttachmentHashCache( + new File( + new File(this.context.getFilesDir(), ATTACHMENT_CACHE_DIR), + HASH_CACHE_FILE)); } /** @@ -165,28 +180,34 @@ public SnapshotBuildResult buildSnapshot() throws IOException { ensureSeeded(); List records = new ArrayList<>(); List problems = new ArrayList<>(); - for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { - if (metadata.deletedAt != null) { - records.add( - SyncRecord.tombstone( - SyncRecord.Type.fromWireValue(metadata.recordType), - metadata.stableId, - Instant.ofEpochMilli(metadata.updatedAt), - Instant.ofEpochMilli(metadata.deletedAt))); - continue; - } - JsonObject payload = payload(metadata, problems); - if (payload != null) { - SyncMetadataEntity current = - database.syncMetadataDao().get(metadata.recordType, metadata.localId); - records.add( - SyncRecord.live( - SyncRecord.Type.fromWireValue(metadata.recordType), - metadata.stableId, - Instant.ofEpochMilli( - current == null ? metadata.updatedAt : current.updatedAt), - payload)); + try { + for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { + if (metadata.deletedAt != null) { + records.add( + SyncRecord.tombstone( + SyncRecord.Type.fromWireValue(metadata.recordType), + metadata.stableId, + Instant.ofEpochMilli(metadata.updatedAt), + Instant.ofEpochMilli(metadata.deletedAt))); + continue; + } + JsonObject payload = payload(metadata, problems); + if (payload != null) { + SyncMetadataEntity current = + database.syncMetadataDao().get(metadata.recordType, metadata.localId); + records.add( + SyncRecord.live( + SyncRecord.Type.fromWireValue(metadata.recordType), + metadata.stableId, + Instant.ofEpochMilli( + current == null + ? metadata.updatedAt + : current.updatedAt), + payload)); + } } + } finally { + hashCache.flush(); } SyncSnapshot snapshot = new SyncSnapshot(records); return problems.isEmpty() @@ -216,6 +237,16 @@ private void applySnapshotInternal( @Nullable SyncState finalState) throws IOException { PreferencesBackup stagedPreferences = selectedPreferences(snapshot); + if (stagedPreferences != null && preferencesChangedSinceBuild()) { + // The settings screens write SharedPreferences directly and nothing touches the sync + // record for them, so the stale-record guard below cannot protect a setting changed + // while the sync was in flight. Committing the merged version would overwrite it and + // record its digest as the baseline, hiding the loss from the next build. Leaving the + // live values alone means the next build sees them differ from the baseline and + // publishes them as the local edit they are. + Log.w(TAG, "Skipping synchronized preferences; they were edited during this sync"); + stagedPreferences = null; + } String stagedPreferencesJson = stagedPreferences == null ? null : gson.toJson(stagedPreferences); String stagedPreferencesTarget = @@ -226,6 +257,7 @@ private void applySnapshotInternal( long stagedPreferencesUpdatedAt = preferencesRecord == null ? 0L : preferencesRecord.getUpdatedAt().toEpochMilli(); boolean deferFinalState = stagedPreferences != null && finalState != null; + boolean applyPreferences = stagedPreferences != null; try { database.runInTransaction( () -> { @@ -233,15 +265,15 @@ private void applySnapshotInternal( Map byStableId = new HashMap<>(); for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { - byStableId.put( - metadata.recordType + ":" + metadata.stableId, metadata); + byStableId.put(recordKey(metadata), metadata); } + // Records the merge decided about but this apply left untouched. A + // conflict for one of them names versions that no longer describe + // the local record, so it must not be stored for the user to apply. + Set skippedKeys = new HashSet<>(); for (SyncRecord record : snapshot.getRecords()) { - SyncMetadataEntity metadata = - byStableId.get( - record.getType().getWireValue() - + ":" - + record.getId()); + String key = recordKey(record); + SyncMetadataEntity metadata = byStableId.get(key); if (metadata == null && !record.isTombstone()) { long localId = insertRemoteRecord(record); if (localId >= 0) { @@ -259,14 +291,20 @@ private void applySnapshotInternal( continue; } if (metadata == null) continue; + if (record.getType() == SyncRecord.Type.PREFERENCES + && !record.isTombstone() + && !applyPreferences) { + // Invalid payload or edited mid-sync: the version is not + // going to be committed, so it must not be recorded as the + // one this device holds either. + skippedKeys.add(key); + continue; + } // The snapshot was built before Drive was read and every blob - // transferred, which - // can take minutes, and the six-hourly worker does it while the - // user is in the - // editor. If the record moved on locally since then, the merge - // chose between - // versions one of which no longer exists, so applying its result - // would silently + // transferred, which can take minutes, and the six-hourly worker + // does it while the user is in the editor. If the record moved on + // locally since then, the merge chose between versions one of + // which no longer exists, so applying its result would silently // drop the newer edit. Leave it; the next sync merges the real // current version. if (metadata.updatedAt > record.getUpdatedAt().toEpochMilli()) { @@ -275,6 +313,7 @@ private void applySnapshotInternal( "Skipping a stale sync result for " + metadata.recordType + "; it was edited during this sync"); + skippedKeys.add(key); continue; } if (record.isTombstone()) { @@ -297,7 +336,7 @@ private void applySnapshotInternal( null); transactionFailureInjector.afterRecordApplied(record); } - persistConflicts(conflicts); + persistConflicts(conflicts, skippedKeys); if (stagedPreferencesJson != null) { database.syncPendingPreferencesDao() .upsert( @@ -336,6 +375,27 @@ private void applySnapshotInternal( pruneAttachmentCache(snapshot); } + @NonNull + private static String recordKey(@NonNull SyncMetadataEntity metadata) { + return metadata.recordType + ":" + metadata.stableId; + } + + @NonNull + private static String recordKey(@NonNull SyncRecord record) { + return record.getType().getWireValue() + ":" + record.getId(); + } + + /** + * Whether the live settings differ from what the snapshot was built from. + * + *

Every build records the live digest as the baseline, so a baseline that no longer matches + * means the user changed a setting after the build and before this apply. + */ + private boolean preferencesChangedSinceBuild() { + String baseline = preferences.getString(PREFERENCES_HASH, null); + return baseline != null && !baseline.equals(livePreferencesDigest()); + } + /** * Drops cached blobs nothing can still need. * @@ -346,7 +406,9 @@ private void applySnapshotInternal( * deleting the bytes would be unrecoverable. * *

Best effort by design: this is a space optimization, and correctness must not depend on it - * running, or on it finishing. + * running, or on it finishing. An unreadable conflict row throws out of the collection below, + * which lands here and skips the whole pass: a row that cannot be read must never authorize a + * deletion. */ private void pruneAttachmentCache(@NonNull SyncSnapshot applied) { try { @@ -355,14 +417,13 @@ private void pruneAttachmentCache(@NonNull SyncSnapshot applied) { collectConflictAttachmentHashes(conflict.winnerJson, required); collectConflictAttachmentHashes(conflict.loserJson, required); } - File dir = new File(context.getFilesDir(), "sync-attachments"); - File[] cached = dir.listFiles(); + File[] cached = attachmentCacheDir().listFiles(); if (cached == null) { return; } for (File file : cached) { String name = file.getName(); - if (!file.isFile() || !name.matches("[0-9a-f]{64}") || required.contains(name)) { + if (!file.isFile() || !name.matches(SHA_256) || required.contains(name)) { continue; } if (!file.delete()) { @@ -380,44 +441,43 @@ private void collectConflictAttachmentHashes( if (recordJson == null || recordJson.isEmpty()) { return; } - try { - JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); - JsonObject payload = root.getAsJsonObject("payload"); - if (payload == null) { - return; - } - JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); - if (manifest != null) { - for (JsonElement element : manifest) { - if (!element.isJsonObject()) continue; - JsonObject entry = element.getAsJsonObject(); - if (entry.has("sha256")) into.add(entry.get("sha256").getAsString()); - } - } - JsonArray hashes = payload.getAsJsonArray("attachmentHashes"); - if (hashes != null) { - for (JsonElement element : hashes) into.add(element.getAsString()); + JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); + JsonObject payload = root.getAsJsonObject("payload"); + if (payload == null) { + return; + } + JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); + if (manifest != null) { + for (JsonElement element : manifest) { + if (!element.isJsonObject()) continue; + JsonObject entry = element.getAsJsonObject(); + if (entry.has("sha256")) into.add(entry.get("sha256").getAsString()); } - } catch (RuntimeException unreadable) { - // An unreadable conflict row must never authorize a deletion, so fail closed by - // keeping everything: the caller only removes blobs nothing claimed. - throw unreadable; + } + JsonArray hashes = payload.getAsJsonArray("attachmentHashes"); + if (hashes != null) { + for (JsonElement element : hashes) into.add(element.getAsString()); } } + /** + * The preferences version this apply should commit, or {@code null} when there is none. + * + *

An unusable payload is skipped rather than thrown: it came from Drive, every device sees + * the same one, and throwing here failed every sync on every device until somebody happened to + * change a setting locally. Skipping leaves the local record's version untouched, so the local + * values keep winning the next merge and the bad version is replaced the moment any device + * publishes a real one. + */ @Nullable - private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) - throws IOException { + private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) { SyncRecord record = snapshot.find(SyncRecord.Type.PREFERENCES, PREFERENCES_STABLE_ID); if (record == null || record.isTombstone()) return null; try { - PreferencesBackup parsed = gson.fromJson(record.getPayload(), PreferencesBackup.class); - if (parsed == null || !parsed.isCreated()) { - throw new IOException("Sync preferences payload is invalid"); - } - return parsed; - } catch (RuntimeException error) { - throw new IOException("Sync preferences payload is invalid", error); + return requirePreferences(record.getPayload()); + } catch (IOException invalid) { + Log.w(TAG, "Skipping an unreadable synchronized preferences payload", invalid); + return null; } } @@ -539,16 +599,25 @@ private void commitPendingPreferences( private void noteLocalPreferenceEdit( @NonNull SyncMetadataEntity metadata, @Nullable PreferencesBackup live) { String digest = preferencesDigest(live); - String baseline = preferences.getString(PREFERENCES_HASH, null); - if (digest.equals(baseline)) { - return; + PreferencesBaselineDecision.Action action = + PreferencesBaselineDecision.decide( + preferences.getString(PREFERENCES_HASH, null), + digest, + legacyPreferencesFingerprint(live)); + switch (action) { + case UNCHANGED: + return; + case MIGRATE_BASELINE: + // 2.6.49 stored the payload JSON here. Same values, older spelling: rewrite it + // without touching the record, or every upgraded device claims a local edit. + preferences.edit().putString(PREFERENCES_HASH, digest).commit(); + return; + case LOCAL_EDIT: + default: + database.syncMetadataDao() + .touch(metadata.recordType, metadata.localId, System.currentTimeMillis()); + preferences.edit().putString(PREFERENCES_HASH, digest).commit(); } - // No baseline at all means sync has never seen these settings; treating them as a local - // edit is the conservative reading, because the alternative silently loses a fresh - // install's configuration to an older version already on Drive. - database.syncMetadataDao() - .touch(metadata.recordType, metadata.localId, System.currentTimeMillis()); - preferences.edit().putString(PREFERENCES_HASH, digest).commit(); } /** Digest of the preferences currently visible to the app. */ @@ -560,12 +629,13 @@ private String livePreferencesDigest() { /** Stable digest of one preferences payload, used for the journal and the build baseline. */ @NonNull private String preferencesDigest(@Nullable PreferencesBackup backup) { - String json = backup == null ? "" : gson.toJson(backup); - try { - return sha256(new java.io.ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))); - } catch (IOException impossible) { - throw new IllegalStateException("Could not digest preferences", impossible); - } + return Sha256.of(backup == null ? "" : gson.toJson(backup)); + } + + /** The baseline exactly as 2.6.49 wrote it: the payload object's own JSON. */ + @NonNull + private String legacyPreferencesFingerprint(@Nullable PreferencesBackup backup) { + return backup == null ? "" : gson.toJsonTree(backup).getAsJsonObject().toString(); } /** Reads a preferences payload, refusing anything that is not a usable settings snapshot. */ @@ -612,7 +682,8 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { } } if ("note".equals(metadata.recordType) - && !addAttachmentMetadata(result, metadata, snapshotProblems)) { + && !addAttachmentMetadata( + result, metadata, ((Note) value).getTitle(), snapshotProblems)) { return null; } // Runs last: the blocks above still need the local categoryId and attachment paths. @@ -621,6 +692,11 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { } private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throws IOException { + // A record that was deleted here and then edited on another device has no row left to + // update; @Update on a missing row is a silent no-op, and the tombstone was cleared + // regardless, so tasks, categories and tags marked live never came back. The REPLACE + // inserts put the row back under its own local id. + boolean revive = metadata.deletedAt != null; if ("note".equals(metadata.recordType)) { Note note = gson.fromJson(payload, Note.class); note.setId((int) metadata.localId); @@ -637,15 +713,27 @@ private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throw payload.get("categoryStableId").getAsString()); if (category != null) task.setCategoryId((int) category.localId); } - database.taskDao().updateTask(task); + if (revive || database.taskDao().getTaskSync(task.getId()) == null) { + database.taskDao().insertTask(task); + } else { + database.taskDao().updateTask(task); + } } else if (SyncMetadata.RECORD_TYPE_CATEGORY.equals(metadata.recordType)) { TaskCategory category = gson.fromJson(payload, TaskCategory.class); category.setId((int) metadata.localId); - database.taskCategoryDao().updateCategory(category); + if (revive || database.taskCategoryDao().getCategorySync(category.getId()) == null) { + database.taskCategoryDao().insertCategory(category); + } else { + database.taskCategoryDao().updateCategory(category); + } } else if ("tag".equals(metadata.recordType)) { Tag tag = gson.fromJson(payload, Tag.class); tag.id = metadata.localId; - database.tagsDao().updateTag(tag); + if (revive || database.tagsDao().getTagSync(tag.id) == null) { + database.tagsDao().addTag(tag); + } else { + database.tagsDao().updateTag(tag); + } } else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { // SharedPreferences is outside Room. applySnapshotInternal journals and commits this // payload only after the Room transaction succeeds. @@ -690,13 +778,50 @@ private long insertRemoteRecord(SyncRecord record) throws IOException { return database.taskCategoryDao().insertCategory(category); } if (record.getType() == SyncRecord.Type.TAG) { - Tag tag = gson.fromJson(record.getPayload(), Tag.class); - tag.id = 0; - return database.tagsDao().addTag(tag); + return insertRemoteTag(record); } return -1; } + /** + * Inserts a tag another device created, unless this device already has it by name. + * + *

A note stores its tag by name and the table has no unique index on it, so a tag created as + * "Work" on two devices before their first sync arrived here as two stable ids and became two + * rows the user saw twice. The name is the tag's real identity; the two stable ids are + * reconciled deterministically so every device ends up with the same one: the smaller id wins, + * the loser's row is deleted and tombstoned, and the tombstone retires the other id on every + * device at the next sync. The device already holding the winning id simply keeps it. + * + * @return the local row bound to the record's stable id, or -1 when the record is skipped. + */ + private long insertRemoteTag(@NonNull SyncRecord record) { + Tag tag = gson.fromJson(record.getPayload(), Tag.class); + tag.id = 0; + String name = tag.getNameTag(); + Tag existing = + name == null || name.isEmpty() ? null : database.tagsDao().getTagByNameSync(name); + if (existing == null) { + return database.tagsDao().addTag(tag); + } + SyncMetadataEntity existingMetadata = + database.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, existing.getId()); + if (existingMetadata == null) { + // A row sync has never described; the remote identity becomes its identity. + return existing.getId(); + } + if (existingMetadata.stableId.compareTo(record.getId()) < 0) { + // This device holds the winning identity; the other one is retired by whichever + // device holds it once it sees ours. + return -1; + } + database.tagsDao().deleteById(existing.getId()); + database.syncMetadataDao() + .markDeleted( + SyncMetadata.RECORD_TYPE_TAG, existing.getId(), System.currentTimeMillis()); + return database.tagsDao().addTag(tag); + } + private void markDeleted(SyncMetadataEntity metadata) { if ("note".equals(metadata.recordType)) database.noteDao().deleteById((int) metadata.localId); @@ -726,28 +851,39 @@ public List getConflicts() { * *

Record identity in {@code sync_metadata} is deliberately kept: it is local, and discarding * it would make the whole library look brand new to the next account. What goes is the sync - * status, the conflict queue and the blobs downloaded from the disconnected account's Drive. + * status, the conflict queue, the preferences journal and the blobs downloaded from the + * disconnected account's Drive. The journal matters: left behind, a fresh store replayed the + * old account's settings onto the device at its next seeding. * *

Clearing the status also repairs a dead end: the Backup screen decided whether to ask for * first-sync consent from {@code lastSuccessfulSyncAt}, which survived a sign-out, while {@code * SyncCoordinator} gated the sync on a preference the sign-out reset. The dialog was skipped * and the sync refused, with no way to reach the consent again. + * + *

Waits for a sync in flight rather than racing it: the six-hourly worker holds the sync + * lock for minutes, and clearing under it left the worker writing the old account's state and + * conflicts back after the wipe, into a cache directory that had just been deleted. */ public void clearAfterDisconnect() { - database.runInTransaction( + SyncService.runWhileNoSyncRuns( () -> { - database.syncStateDao().clear(); - database.syncConflictDao().clearAll(); + database.runInTransaction( + () -> { + database.syncStateDao().clear(); + database.syncConflictDao().clearAll(); + database.syncPendingPreferencesDao().clear(); + }); + preferences.edit().remove(PREFERENCES_HASH).remove(LEGACY_STATE).apply(); + localAttachments.clear(); + noteFoldersIndexed = false; + hashCache.clear(); + deleteAttachmentCache(); }); - preferences.edit().remove(PREFERENCES_HASH).remove(LEGACY_STATE).apply(); - localAttachments.clear(); - deleteAttachmentCache(); } /** Removes the download cache only; the notes' own attachment folders are untouched. */ private void deleteAttachmentCache() { - File dir = new File(context.getFilesDir(), "sync-attachments"); - File[] cached = dir.listFiles(); + File[] cached = attachmentCacheDir().listFiles(); if (cached == null) { return; } @@ -768,6 +904,17 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) SyncConflictEntity pending = database.syncConflictDao().getById(conflictId); if (pending == null || pending.resolved) return; + if (isSuperseded(pending)) { + // The record moved on after this conflict was recorded — an edit here, or a newer + // version applied from another device — so both stored versions are older than what + // the user now has. Applying either would overwrite the newer edit with a version + // that was never offered against it. The alternative still travels with the bundle + // and comes back as a fresh conflict against the current version at the next sync. + Log.w(TAG, "Dropping a conflict that the record's newer version has superseded"); + database.runInTransaction(() -> database.syncConflictDao().deleteById(conflictId)); + return; + } + // Resolution is a user-visible mutation. Verify and pin the selected version before its // conflict row can be marked resolved; a missing blob must leave both the note and the // conflict untouched, including when the winner happens to already be visible in Room. @@ -801,6 +948,14 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) } } + /** True when the local record is newer than both versions the conflict offers. */ + private boolean isSuperseded(@NonNull SyncConflictEntity conflict) { + SyncMetadataEntity metadata = + database.syncMetadataDao().getByStableId(conflict.recordType, conflict.stableId); + return metadata != null + && metadata.updatedAt > Math.max(conflict.winnerUpdatedAt, conflict.loserUpdatedAt); + } + /** * Applies a chosen preferences version through the same journal a snapshot apply uses. * @@ -901,12 +1056,27 @@ private void pinResolvedConflictAttachments(@NonNull SyncRecord selected) throws } } - private void persistConflicts(@NonNull List conflicts) { + /** + * Stores the conflicts this apply produced and retires the ones it makes meaningless. + * + * @param skippedKeys records this apply left untouched; a conflict for one of them offers + * versions that no longer describe the local record and is not stored. + */ + private void persistConflicts( + @NonNull List conflicts, @NonNull Set skippedKeys) { if (conflicts.isEmpty()) return; long createdAt = System.currentTimeMillis(); List rows = new ArrayList<>(conflicts.size()); + Map> winnersByRecord = new HashMap<>(); for (SyncMergeResult.Conflict conflict : conflicts) { + String key = conflict.getType().getWireValue() + ":" + conflict.getId(); + if (skippedKeys.contains(key)) { + continue; + } + winnersByRecord + .computeIfAbsent(key, ignored -> new LinkedHashSet<>()) + .add(conflict.getWinnerVersionId()); rows.add( new SyncConflictEntity( conflict.getType().getWireValue(), @@ -927,25 +1097,38 @@ private void persistConflicts(@NonNull List conflicts) createdAt, 0L)); } + if (rows.isEmpty()) return; + // Every conflict for a record names the version this sync applies as its winner. An + // older unresolved row for the same record therefore offers a winner that is no longer + // the live version, and applying it — which "keep the version the merge selected" did + // with one tap — reverted the user's newer edit. Its alternative is not lost: it still + // travels with the bundle and is among the rows stored here, now against the current + // version. + for (SyncMergeResult.Conflict conflict : conflicts) { + String key = conflict.getType().getWireValue() + ":" + conflict.getId(); + Set winners = winnersByRecord.remove(key); + if (winners == null || winners.size() != 1) { + continue; + } + database.syncConflictDao() + .deleteSupersededUnresolved( + conflict.getType().getWireValue(), + conflict.getId(), + winners.iterator().next()); + } database.syncConflictDao().insertIgnoringDuplicates(rows); } @NonNull private static String conflictVersionPairHash(@NonNull SyncMergeResult.Conflict conflict) { - String source = + return Sha256.of( conflict.getType().getWireValue() + "\n" + conflict.getId() + "\n" + conflict.getWinner().canonicalSerializedPayload() + "\n" - + conflict.getLoser().canonicalSerializedPayload(); - try { - return sha256( - new java.io.ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); - } catch (IOException impossible) { - throw new IllegalStateException("Could not hash sync conflict identity", impossible); - } + + conflict.getLoser().canonicalSerializedPayload()); } /** True when {@code resolution} names the version the merge selected. */ @@ -1090,6 +1273,12 @@ public boolean hasAttachment(@NonNull String sha256) { return resolveLocalAttachment(sha256) != null; } + @Override + public boolean hasDurableAttachment(@NonNull String sha256, long sizeBytes) { + File cached = attachmentFile(sha256); + return cached.isFile() && (sizeBytes < 0L || cached.length() == sizeBytes); + } + @NonNull @Override public InputStream readAttachment(@NonNull String sha256) throws IOException { @@ -1110,7 +1299,9 @@ public InputStream readAttachment(@NonNull String sha256) throws IOException { * this returns true, that failure was permanent for any account holding a single attachment. * *

The note folders are indexed while the snapshot is built rather than copied into the - * cache, so a large attachment set is not stored twice. + * cache, so a large attachment set is not stored twice. A store that never built a snapshot — + * the Backup screen's, resolving a conflict the worker's store found — indexes them on its + * first miss instead, so a version whose blob sits in a note folder is not reported missing. */ @Nullable private File resolveLocalAttachment(@NonNull String sha256) { @@ -1119,9 +1310,42 @@ private File resolveLocalAttachment(@NonNull String sha256) { return cached; } File owned = localAttachments.get(sha256); + if (owned != null && owned.isFile()) { + return owned; + } + if (noteFoldersIndexed) { + return null; + } + indexNoteFolders(); + owned = localAttachments.get(sha256); return owned != null && owned.isFile() ? owned : null; } + /** Hashes every file under every note folder into {@link #localAttachments}, once. */ + private synchronized void indexNoteFolders() { + if (noteFoldersIndexed) { + return; + } + File[] folders = AttachmentStorage.baseDirPath(context).listFiles(); + if (folders != null) { + for (File folder : folders) { + File[] files = folder.isDirectory() ? folder.listFiles() : null; + if (files == null) continue; + for (File file : files) { + if (!file.isFile()) continue; + try { + localAttachments.putIfAbsent( + hashCache.sha256(file, attachmentHasher::sha256), file); + } catch (IOException | RuntimeException unreadable) { + // A file that cannot be hashed cannot satisfy a reference either. + } + } + } + } + hashCache.flush(); + noteFoldersIndexed = true; + } + @Override public void writeAttachment( @NonNull String sha256, long sizeBytes, @NonNull InputStream content) @@ -1145,15 +1369,31 @@ public void writeAttachment( if (!temp.renameTo(target)) throw new IOException("Cannot store attachment"); } + private File attachmentCacheDir() { + return new File(context.getFilesDir(), ATTACHMENT_CACHE_DIR); + } + private File attachmentFile(String sha256) { - File dir = new File(context.getFilesDir(), "sync-attachments"); + File dir = attachmentCacheDir(); if (!dir.exists()) dir.mkdirs(); return new File(dir, sha256); } + /** + * Describes a note's attachments for the wire and puts its editor blocks into wire form. + * + *

The attachments column names files by this device's row id and file name; the wire + * describes each attachment by a logical id, its content hash and size. A file that is gone + * from disk is still describable when the column remembers its hash and size — which every + * attachment that ever came through a sync does — so such a note is published from that + * metadata and {@link SyncService} fetches the bytes from the cache or Drive; the following + * apply then puts the file back. Only an attachment nothing but this device ever knew about + * fails the build, and then the problem names the note. + */ private boolean addAttachmentMetadata( JsonObject payload, SyncMetadataEntity metadata, + @Nullable String noteTitle, @NonNull List snapshotProblems) { String json = payload.has("h") && !payload.get("h").isJsonNull() @@ -1165,43 +1405,38 @@ private boolean addAttachmentMetadata( attachments = JsonParser.parseString(json).getAsJsonArray(); } catch (RuntimeException error) { addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, metadata); + snapshotProblems, + SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, + metadata, + noteTitle); return false; } JsonArray manifest = new JsonArray(); JsonArray hashes = new JsonArray(); JsonObject names = new JsonObject(); + Map logicalIdByUrl = new HashMap<>(); boolean complete = true; for (int attachmentIndex = 0; attachmentIndex < attachments.size(); attachmentIndex++) { JsonElement element = attachments.get(attachmentIndex); - if (!element.isJsonObject()) { - addSnapshotProblem( - snapshotProblems, - SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); - complete = false; - continue; - } - EditorAttachment attachment; - try { - attachment = gson.fromJson(element, EditorAttachment.class); - } catch (RuntimeException error) { - addSnapshotProblem( - snapshotProblems, - SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); - complete = false; - continue; + EditorAttachment attachment = null; + if (element.isJsonObject()) { + try { + attachment = gson.fromJson(element, EditorAttachment.class); + } catch (RuntimeException error) { + attachment = null; + } } if (attachment == null || attachment.url == null || attachment.url.trim().isEmpty()) { addSnapshotProblem( snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); + metadata, + noteTitle); complete = false; continue; } + JsonObject column = element.getAsJsonObject(); File file; try { file = attachmentResolver.resolve(context, attachment); @@ -1209,41 +1444,67 @@ private boolean addAttachmentMetadata( addSnapshotProblem( snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); - complete = false; - continue; - } - if (file == null || !file.isFile()) { - addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.MISSING_ATTACHMENT, metadata); - complete = false; - continue; - } - if (!file.canRead()) { - addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.UNREADABLE_ATTACHMENT, metadata); + metadata, + noteTitle); complete = false; continue; } + String rememberedHash = optionalString(column, "sha256"); + long rememberedSize = optionalLong(column, "size"); String hash; - try { - hash = attachmentHasher.sha256(file); - } catch (IOException error) { - addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.ATTACHMENT_HASH_FAILED, metadata); - complete = false; - continue; + long size; + if (file == null || !file.isFile()) { + if (rememberedHash == null + || !rememberedHash.matches(SHA_256) + || rememberedSize < 0L) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.MISSING_ATTACHMENT, + metadata, + noteTitle); + complete = false; + continue; + } + Log.w( + TAG, + "An attachment file is missing; it will be restored from the sync cache" + + " or Drive"); + file = null; + hash = rememberedHash; + size = rememberedSize; + } else { + if (!file.canRead()) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.UNREADABLE_ATTACHMENT, + metadata, + noteTitle); + complete = false; + continue; + } + try { + hash = hashCache.sha256(file, attachmentHasher::sha256); + } catch (IOException error) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.ATTACHMENT_HASH_FAILED, + metadata, + noteTitle); + complete = false; + continue; + } + size = file.length(); + localAttachments.put(hash, file); } - localAttachments.put(hash, file); - String displayName = - attachment.name == null || attachment.name.trim().isEmpty() - ? file.getName() - : attachment.name.trim(); + String displayName = displayNameFor(attachment, file, hash); String logicalId = attachment.id; if (!isCanonicalUuid(logicalId)) { // Existing editor data predates logical attachment IDs. Deriving from the stable - // note, source URL and position keeps the migration deterministic while allowing - // equal-content references to remain distinct logical attachments. + // note, source URL, position and content keeps the migration deterministic while + // allowing equal-content references to remain distinct logical attachments. The + // content hash is part of it so that one id can never describe two different + // blobs: a bundle manifest is keyed by id, and two versions of a note disagreeing + // about an id's content used to fail every publish for the account. // // The check is canonical-UUID rather than a loose 36-character pattern: the // bundle manifest only accepts canonical lowercase UUIDs, so an uppercase or @@ -1257,37 +1518,99 @@ private boolean addAttachmentMetadata( + "\n" + attachment.url + "\n" - + displayName) + + displayName + + "\n" + + hash) .getBytes(StandardCharsets.UTF_8)) .toString(); } hashes.add(hash); names.addProperty(logicalId, displayName); + logicalIdByUrl.put(comparableUrl(attachment.url), logicalId); + String rememberedMimeType = optionalString(column, "mimeType"); JsonObject manifestEntry = new JsonObject(); manifestEntry.addProperty("id", logicalId); manifestEntry.addProperty("sha256", hash); - manifestEntry.addProperty("mimeType", detectMimeType(file, attachment, displayName)); - manifestEntry.addProperty("size", file.length()); + manifestEntry.addProperty( + "mimeType", + rememberedMimeType != null + ? rememberedMimeType + : detectMimeType(file, attachment, displayName)); + manifestEntry.addProperty("size", size); manifestEntry.addProperty("path", "attachments/" + hash); manifestEntry.addProperty("displayName", displayName); manifest.add(manifestEntry); } if (!complete) return false; - if (manifest.size() == 0) { - // A note whose attachments column is "[]" — which is what the editor stores for a - // note that simply has none — used to get three empty arrays here, while a decoded - // remote record carries no attachment fields at all. The two shapes hashed - // differently, so every attachment-free note reported a conflict against itself on - // every sync and republished a bundle each time. - return true; - } + // Empty ones are dropped again by SyncRecord, the one place that rule lives. payload.add("attachmentsManifest", manifest); payload.add("attachmentHashes", hashes); payload.add("attachmentNames", names); + // The blocks name this device's files. On the wire they name the logical attachment, + // which every device agrees on; otherwise the same note hashed differently on every + // device that had ever restored it and conflicted with itself on every sync. + JsonElement valueJson = payload.get("f"); + if (valueJson != null && valueJson.isJsonPrimitive()) { + String local = valueJson.getAsString(); + String wire = + EditorAttachmentBlocks.rewriteUrls( + local, + url -> { + String logicalId = logicalIdByUrl.get(comparableUrl(url)); + return logicalId == null + ? null + : AttachmentWireUrl.forLogicalId(logicalId); + }); + if (wire != null && !wire.equals(local)) { + payload.addProperty("f", wire); + } + } return true; } + /** A URL in the one spelling both the legacy and the canonical scheme reduce to. */ + @NonNull + private static String comparableUrl(@NonNull String url) { + AttachmentUrl parsed = AttachmentUrl.parse(url); + return parsed == null ? url.trim() : parsed.canonical(); + } + + @NonNull + private static String displayNameFor( + @NonNull EditorAttachment attachment, @Nullable File file, @NonNull String hash) { + if (attachment.name != null && !attachment.name.trim().isEmpty()) { + return attachment.name.trim(); + } + if (file != null) { + return file.getName(); + } + AttachmentUrl parsed = AttachmentUrl.parse(attachment.url); + return parsed == null ? hash : parsed.getFileName(); + } + + @Nullable + private static String optionalString(@NonNull JsonObject object, @NonNull String field) { + JsonElement value = object.get(field); + if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) { + return null; + } + String text = value.getAsString().trim(); + return text.isEmpty() ? null : text; + } + + private static long optionalLong(@NonNull JsonObject object, @NonNull String field) { + JsonElement value = object.get(field); + if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) { + return -1L; + } + try { + return value.getAsLong(); + } catch (RuntimeException notALong) { + return -1L; + } + } + /** True only for a lowercase canonical UUID, which is all the bundle manifest accepts. */ private static boolean isCanonicalUuid(@Nullable String value) { if (value == null) { @@ -1303,14 +1626,19 @@ private static boolean isCanonicalUuid(@Nullable String value) { private static void addSnapshotProblem( @NonNull List problems, @NonNull SnapshotProblem.Kind kind, - @NonNull SyncMetadataEntity metadata) { - problems.add(new SnapshotProblem(kind, metadata.recordType, metadata.stableId)); + @NonNull SyncMetadataEntity metadata, + @Nullable String label) { + problems.add(new SnapshotProblem(kind, metadata.recordType, metadata.stableId, label)); } /** * Materializes every attachment before changing the Room row. Targets use the immutable * logical-ID/content-ID pair rather than a display name, so a rollback can leave only harmless * new files and can never alter bytes addressed by the pre-transaction note. + * + *

The column written here remembers the logical id, hash, size and MIME type alongside the + * local file, which is what lets the next build describe the attachment identically to the + * device that sent it — and describe it at all should the file go missing. */ private void restoreAttachments(Note note, JsonObject payload) throws IOException { JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); @@ -1325,6 +1653,7 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio throw new IOException("Could not create attachment folder"); } JsonArray restored = new JsonArray(); + Map localUrlByLogicalId = new HashMap<>(); for (JsonElement element : manifest) { if (!element.isJsonObject()) { throw new IOException("Attachment manifest entry is invalid"); @@ -1338,7 +1667,7 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio if (entry.id == null || entry.sha256 == null || !entry.id.matches("[0-9a-fA-F-]{36}") - || !entry.sha256.matches("[0-9a-f]{64}") + || !entry.sha256.matches(SHA_256) || entry.size < 0L) { throw new IOException("Attachment manifest entry is invalid"); } @@ -1365,16 +1694,22 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio if (note.getId() <= 0) { throw new IOException("Cannot restore attachments for an unsaved note"); } - JsonObject attachment = new JsonObject(); // Canonical editorjs:// form, the only shape EditorAttachmentsWebViewClient serves. // Writing file:// here left every synced attachment unrenderable on the receiver. - attachment.addProperty("url", AttachmentStorage.urlFor(note.getId(), target.getName())); + String url = AttachmentStorage.urlFor(note.getId(), target.getName()); + JsonObject attachment = new JsonObject(); + attachment.addProperty("url", url); attachment.addProperty("name", displayName); attachment.addProperty("id", entry.id); + attachment.addProperty("sha256", entry.sha256); + attachment.addProperty("size", entry.size); + attachment.addProperty("mimeType", entry.mimeType); restored.add(attachment); + localUrlByLogicalId.put(entry.id, url); } note.setAttachments(gson.toJson(restored)); - note.setValueJson(rewriteEditorAttachmentUrls(note.getValueJson(), restored)); + note.setValueJson( + rewriteEditorAttachmentUrls(note.getValueJson(), restored, localUrlByLogicalId)); } /** @@ -1385,54 +1720,54 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio * note_/}. The column is what the file list reads; the blocks are what * the editor renders, so a received rich note showed its attachments as broken. * - *

The mapping is positional, which is exactly how the manifest was built: the sender's - * attachments column comes from {@code EditorJsonUtils} walking these same blocks in document - * order, and {@code addAttachmentMetadata} walks that column in the same order. If the two do - * not line up the JSON is returned untouched rather than guessed at — a note that renders the - * old broken URL is recoverable, one whose content was rewritten wrongly is not. + *

A block written by this release names its attachment's logical id, which maps directly. A + * bundle from an older client still names the sender's files, and for those the mapping is + * positional, which is exactly how such a manifest was built: the sender's attachments column + * came from {@code EditorJsonUtils} walking these same blocks in document order, and the + * manifest walked that column in the same order. If the two do not line up the JSON is returned + * untouched rather than guessed at — a note that renders the old broken URL is recoverable, one + * whose content was rewritten wrongly is not. */ @Nullable private String rewriteEditorAttachmentUrls( - @Nullable String valueJson, @NonNull JsonArray restored) { + @Nullable String valueJson, + @NonNull JsonArray restored, + @NonNull Map localUrlByLogicalId) { if (valueJson == null || valueJson.trim().isEmpty() || restored.size() == 0) { return valueJson; } - try { - JsonArray blocks = JsonParser.parseString(valueJson).getAsJsonArray(); - List files = new ArrayList<>(); - for (JsonElement element : blocks) { - if (!element.isJsonObject()) continue; - JsonObject block = element.getAsJsonObject(); - String type = - block.has("type") && block.get("type").isJsonPrimitive() - ? block.get("type").getAsString() - : ""; - if (!"attaches".equals(type) && !"image".equals(type)) continue; - JsonObject data = block.getAsJsonObject("data"); - if (data == null) continue; - JsonObject file = data.getAsJsonObject("file"); - if (file != null) files.add(file); - } - if (files.size() != restored.size()) { - Log.w(TAG, "Editor blocks do not match the restored attachments; leaving them"); - return valueJson; - } - for (int index = 0; index < files.size(); index++) { - JsonObject target = restored.get(index).getAsJsonObject(); - files.get(index).addProperty("url", target.get("url").getAsString()); - files.get(index).addProperty("name", target.get("name").getAsString()); + List urls = EditorAttachmentBlocks.fileUrls(valueJson); + boolean wireForm = false; + for (String url : urls) { + if (AttachmentWireUrl.logicalIdOf(url) != null) { + wireForm = true; + break; } - return gson.toJson(blocks); - } catch (RuntimeException malformed) { - Log.w(TAG, "Could not rewrite editor attachment URLs; leaving them untouched"); + } + if (wireForm) { + return EditorAttachmentBlocks.rewriteUrls( + valueJson, + url -> { + String logicalId = AttachmentWireUrl.logicalIdOf(url); + return logicalId == null ? null : localUrlByLogicalId.get(logicalId); + }); + } + if (urls.size() != restored.size()) { + Log.w(TAG, "Editor blocks do not match the restored attachments; leaving them"); return valueJson; } + int[] position = {0}; + return EditorAttachmentBlocks.rewriteUrls( + valueJson, + url -> restored.get(position[0]++).getAsJsonObject().get("url").getAsString()); } - private static boolean isVerifiedAttachmentFile( + private boolean isVerifiedAttachmentFile( @NonNull File file, @NonNull String expectedHash, long expectedSize) throws IOException { - return file.isFile() && file.length() == expectedSize && expectedHash.equals(sha256(file)); + return file.isFile() + && file.length() == expectedSize + && expectedHash.equals(hashCache.sha256(file, Sha256::of)); } private static void copyVerifiedAttachment( @@ -1443,39 +1778,22 @@ private static void copyVerifiedAttachment( throws IOException { File temporary = new File(target.getParentFile(), target.getName() + ".tmp-" + UUID.randomUUID()); - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - long copied = 0L; - try (InputStream in = new FileInputStream(source); + try (VerifyingInputStream in = + new VerifyingInputStream( + new FileInputStream(source), expectedHash, expectedSize); OutputStream out = new FileOutputStream(temporary)) { byte[] buffer = new byte[8192]; int read; while ((read = in.read(buffer)) != -1) { out.write(buffer, 0, read); - digest.update(buffer, 0, read); - copied += read; - if (copied > expectedSize) { - throw new AttachmentIntegrityException("Attachment exceeds its declared size"); - } } + in.verifyEndOfStream(); } catch (IOException failure) { if (temporary.exists() && !temporary.delete()) { Log.w(TAG, "Could not remove failed staged attachment"); } throw failure; } - StringBuilder hash = new StringBuilder(64); - for (byte value : digest.digest()) hash.append(String.format("%02x", value & 0xff)); - String actual = hash.toString(); - if (copied != expectedSize || !expectedHash.equals(actual)) { - if (!temporary.delete()) Log.w(TAG, "Could not remove invalid staged attachment"); - throw new AttachmentIntegrityException( - "Attachment checksum does not match sync metadata"); - } if (!temporary.renameTo(target)) { if (!temporary.delete()) Log.w(TAG, "Could not remove uncommitted staged attachment"); throw new IOException("Could not finalize staged attachment"); @@ -1483,51 +1801,7 @@ private static void copyVerifiedAttachment( } private static boolean isSafeAttachmentName(@NonNull String name) { - String value = name.trim(); - if (value.isEmpty() || value.equals(".") || value.equals("..")) return false; - if (value.length() > 255 || value.contains("/") || value.contains("\\")) return false; - if (value.contains("..") || value.indexOf('\u0000') >= 0) return false; - for (int i = 0; i < value.length(); i++) { - if (Character.isISOControl(value.charAt(i))) return false; - } - return new File(value).getName().equals(value); - } - - private static String sha256(File file) throws IOException { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - try (InputStream in = new FileInputStream(file)) { - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) != -1) digest.update(buffer, 0, read); - } - StringBuilder hex = new StringBuilder(64); - for (byte value : digest.digest()) hex.append(String.format("%02x", value & 0xff)); - return hex.toString(); - } - - @NonNull - private static String sha256(@NonNull InputStream input) throws IOException { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - try (InputStream in = input) { - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) != -1) { - digest.update(buffer, 0, read); - } - } - StringBuilder hex = new StringBuilder(64); - for (byte value : digest.digest()) hex.append(String.format("%02x", value & 0xff)); - return hex.toString(); + return AttachmentUrl.isSafeSegment(name.trim()); } /** Resolves a serialized note attachment to its app-private file. */ @@ -1549,7 +1823,7 @@ public interface TransactionFailureInjector { @NonNull private static String detectMimeType( - @NonNull File file, EditorAttachment attachment, @NonNull String displayName) { + @Nullable File file, EditorAttachment attachment, @NonNull String displayName) { String mimeType = URLConnection.guessContentTypeFromName(displayName); if (mimeType != null && !mimeType.trim().isEmpty()) { return mimeType; @@ -1566,7 +1840,7 @@ private static String detectMimeType( } } - mimeType = URLConnection.guessContentTypeFromName(file.getName()); + mimeType = file == null ? null : URLConnection.guessContentTypeFromName(file.getName()); return mimeType == null || mimeType.trim().isEmpty() ? "application/octet-stream" : mimeType; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java b/app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java new file mode 100644 index 00000000..52b77eb1 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java @@ -0,0 +1,77 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * The one SHA-256 the sync code hashes with. + * + *

Five copies of the digest-and-hex loop used to live in the store, the validator, the service + * and the backend, three of them formatting through {@code String.format("%02x")} with the default + * locale. A blob hashed by one copy has to match a manifest written by another, so the hex encoding + * is a lookup table here and nowhere else. + */ +final class Sha256 { + + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + private Sha256() {} + + /** Lowercase hex, locale-independent by construction. */ + @NonNull + static String hex(@NonNull byte[] bytes) { + char[] out = new char[bytes.length * 2]; + for (int index = 0; index < bytes.length; index++) { + int value = bytes[index] & 0xff; + out[index * 2] = HEX[value >>> 4]; + out[index * 2 + 1] = HEX[value & 0xf]; + } + return new String(out); + } + + @NonNull + static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException missing) { + // Mandatory on every Java platform; treating it as recoverable only hides a broken + // runtime behind a sync error. + throw new IllegalStateException("SHA-256 is unavailable", missing); + } + } + + @NonNull + static String of(@NonNull byte[] bytes) { + return hex(newDigest().digest(bytes)); + } + + @NonNull + static String of(@NonNull String value) { + return of(value.getBytes(StandardCharsets.UTF_8)); + } + + /** Reads {@code input} to its end without closing it. */ + @NonNull + static String of(@NonNull InputStream input) throws IOException { + MessageDigest digest = newDigest(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + digest.update(buffer, 0, read); + } + return hex(digest.digest()); + } + + @NonNull + static String of(@NonNull File file) throws IOException { + try (InputStream input = new FileInputStream(file)) { + return of(input); + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java index 566bb9d1..25dd77e0 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java @@ -61,7 +61,9 @@ public static final class SnapshotBuildException extends IOException { @NonNull private final List problems; private SnapshotBuildException(@NonNull List problems) { - super("Local snapshot is incomplete: " + problems.get(0).getKind().name()); + // The first problem names its record: a user who reads this on the account screen + // has to know which note to open, not only that some attachment somewhere is gone. + super("Local snapshot is incomplete: " + problems.get(0).describe()); this.problems = Collections.unmodifiableList(new ArrayList<>(problems)); } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java index 6d8aeb9d..daba7b30 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java @@ -1,6 +1,7 @@ package com.pasich.mynotes.data.sync; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.util.Objects; /** A privacy-safe reason why a local snapshot cannot safely be published. */ @@ -13,15 +14,32 @@ public enum Kind { INVALID_ATTACHMENT_METADATA } + private static final int MAX_LABEL_LENGTH = 40; + @NonNull private final Kind kind; @NonNull private final String recordType; @NonNull private final String stableId; + @Nullable private final String label; public SnapshotProblem( @NonNull Kind kind, @NonNull String recordType, @NonNull String stableId) { + this(kind, recordType, stableId, null); + } + + /** + * @param label what the user calls the record — a note's title — so the failure names the note + * to fix instead of leaving them to guess. Stays on the device: it is shown on the account + * screen and kept in the local sync state, never published. + */ + public SnapshotProblem( + @NonNull Kind kind, + @NonNull String recordType, + @NonNull String stableId, + @Nullable String label) { this.kind = Objects.requireNonNull(kind, "kind"); this.recordType = Objects.requireNonNull(recordType, "recordType"); this.stableId = Objects.requireNonNull(stableId, "stableId"); + this.label = label == null || label.trim().isEmpty() ? null : truncate(label.trim()); } @NonNull @@ -38,4 +56,23 @@ public String getRecordType() { public String getStableId() { return stableId; } + + @Nullable + public String getLabel() { + return label; + } + + /** One line naming the failure and, when known, the record it is in. */ + @NonNull + public String describe() { + return label == null + ? kind.name() + : kind.name() + " in " + recordType + " \"" + label + "\""; + } + + private static String truncate(String value) { + return value.length() <= MAX_LABEL_LENGTH + ? value + : value.substring(0, MAX_LABEL_LENGTH) + "…"; + } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java index 74b04833..c8f6e14b 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java @@ -10,9 +10,14 @@ * *

Implementations may use Google Drive, WebDAV, or an on-device test store. Attachments are * addressed by their content SHA-256, so writing the same attachment more than once must be safe. - * {@link #writeSnapshot(SyncSnapshot)} is the commit point: implementations must not make a new + * {@link #publish(SyncPublication)} is the commit point: implementations must not make a new * manifest visible before the call succeeds. The service uploads every required attachment before * it calls this method. + * + *

There is deliberately no plain "read a snapshot" or "write a snapshot" pair. A publish has to + * quote the read it was derived from, or a backend that keeps causal history cannot tell a write + * with stale parents from a legitimate one; the Drive backend had to disable exactly such a + * default, and a future backend must not be able to inherit it by accident. */ public interface SyncBackend { @@ -21,36 +26,21 @@ public interface SyncBackend { String getIdentifier(); /** - * Reads the current remote manifest. + * Reads the remote causal frontier. * - *

Returns {@link SyncSnapshot#empty()} when this backend has not yet received a sync bundle. - */ - @NonNull - SyncSnapshot readSnapshot() throws IOException; - - /** - * Reads the remote causal frontier. Legacy adapters expose one snapshot and no remote - * conflicts; Drive overrides this so concurrent immutable bundle heads remain recoverable. + *

Returns an empty snapshot, with a read token the next publish can quote, when this backend + * has not yet received a sync bundle. */ @NonNull - default RemoteSnapshot readSnapshotResult() throws IOException { - return RemoteSnapshot.of(readSnapshot()); - } - - /** Publishes a complete remote snapshot. Implementations must not expose a partial snapshot. */ - void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException; + RemoteSnapshot readSnapshotResult() throws IOException; /** * Publishes a snapshot together with the unresolved conflict versions it must keep alive and * the read context it was derived from. * - *

Backends that keep no causal history fall back to the plain snapshot write; the Drive - * backend overrides this so a publish cannot use stale causal parents and cannot drop an - * unresolved alternative on the floor. + *

Implementations must refuse a publication whose read context is not their latest read. */ - default void publish(@NonNull SyncPublication publication) throws IOException { - writeSnapshot(publication.getSnapshot()); - } + void publish(@NonNull SyncPublication publication) throws IOException; /** Returns true when the immutable attachment blob already exists remotely. */ boolean hasAttachment(@NonNull String sha256) throws IOException; @@ -68,43 +58,26 @@ default boolean hasVerifiedAttachment(@NonNull String sha256, @Nullable Long exp if (content == null) { return false; } - java.security.MessageDigest digest; - try { - digest = java.security.MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - long size = 0L; try (InputStream input = content) { - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - digest.update(buffer, 0, read); - size += read; - } - } - StringBuilder actual = new StringBuilder(64); - for (byte value : digest.digest()) { - actual.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); + VerifyingInputStream.verify(input, sha256, expectedSize); + return true; + } catch (AttachmentIntegrityException corrupt) { + return false; } - return sha256.equals(actual.toString()) && (expectedSize == null || expectedSize == size); } /** * Opens an attachment by its lowercase SHA-256 hash, or returns {@code null} when it is absent. - * The caller closes the returned stream. + * The caller closes the returned stream and is responsible for verifying it. */ @Nullable InputStream readAttachment(@NonNull String sha256) throws IOException; /** - * Stores a complete attachment under {@code sha256}. + * Stores one immutable blob, streaming it rather than holding it in memory. * *

The implementation must consume the stream before returning and must not expose a partial * file after an exception. - */ - /** - * Stores one immutable blob, streaming it rather than holding it in memory. * * @param sizeBytes the blob's declared size, or a negative value when it is unknown. A known * size lets an implementation avoid buffering the whole blob to compute a content length. diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 98e089c7..2c47218a 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -35,6 +35,10 @@ public final class SyncBundleCodec { public static final String ENTRY_RECORDS = "records.json"; public static final String BUNDLE_FORMAT = "mynotes-sync"; public static final int SCHEMA_VERSION = 1; + + /** Wire field mapping a re-keyed manifest id back to the record's own attachment id. */ + static final String FIELD_ATTACHMENT_ID_ALIASES = "attachmentIdAliases"; + private static final Pattern SHA_256 = Pattern.compile("[0-9a-f]{64}"); private static final Pattern MIME_TYPE = Pattern.compile("^[a-z0-9!#$&^_.+-]+/[a-z0-9!#$&^_.+-]+$"); @@ -82,15 +86,18 @@ public byte[] encode( @NonNull Collection unresolvedAlternatives, @NonNull Collection resolvedAlternativeIds) throws IOException { + List alternatives = dedupeAlternatives(unresolvedAlternatives); + AttachmentPlan attachmentPlan = planAttachments(snapshot, alternatives); JsonObject recordsRoot = new JsonObject(); - recordsRoot.add("notes", liveArray(snapshot, SyncRecord.Type.NOTE)); - recordsRoot.add("tasks", liveArray(snapshot, SyncRecord.Type.TASK)); - recordsRoot.add("tags", liveArray(snapshot, SyncRecord.Type.TAG)); - recordsRoot.add("categories", liveArray(snapshot, SyncRecord.Type.CATEGORY)); - recordsRoot.add("preferences", liveArray(snapshot, SyncRecord.Type.PREFERENCES)); + recordsRoot.add("notes", liveArray(snapshot, SyncRecord.Type.NOTE, attachmentPlan)); + recordsRoot.add("tasks", liveArray(snapshot, SyncRecord.Type.TASK, attachmentPlan)); + recordsRoot.add("tags", liveArray(snapshot, SyncRecord.Type.TAG, attachmentPlan)); + recordsRoot.add( + "categories", liveArray(snapshot, SyncRecord.Type.CATEGORY, attachmentPlan)); + recordsRoot.add( + "preferences", liveArray(snapshot, SyncRecord.Type.PREFERENCES, attachmentPlan)); recordsRoot.add("tombstones", tombstones(snapshot)); - List alternatives = dedupeAlternatives(unresolvedAlternatives); - recordsRoot.add("alternatives", alternativeArray(alternatives)); + recordsRoot.add("alternatives", alternativeArray(alternatives, attachmentPlan)); JsonArray resolved = new JsonArray(); for (String versionId : new java.util.TreeSet<>(resolvedAlternativeIds)) { if (!SHA_256.matcher(versionId).matches()) { @@ -100,7 +107,7 @@ public byte[] encode( } recordsRoot.add("resolvedAlternatives", resolved); - JsonArray attachments = collectAttachments(snapshot, alternatives); + JsonArray attachments = attachmentPlan.manifest(); byte[] recordBytes = GSON.toJson(recordsRoot).getBytes(StandardCharsets.UTF_8); if (recordBytes.length > SyncBundleValidator.MAX_RECORD_BYTES) { throw new IOException("Sync records exceed the schema-1 size limit"); @@ -192,31 +199,46 @@ private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) @NonNull private static JsonArray liveArray( - @NonNull SyncSnapshot snapshot, @NonNull SyncRecord.Type type) throws IOException { + @NonNull SyncSnapshot snapshot, + @NonNull SyncRecord.Type type, + @NonNull AttachmentPlan attachmentPlan) + throws IOException { JsonArray array = new JsonArray(); for (SyncRecord record : snapshot.getLiveRecords(type)) { JsonObject item = record.getPayload(); item.addProperty("id", record.getId()); item.addProperty("updatedAt", record.getUpdatedAt().toString()); if (type == SyncRecord.Type.NOTE) { - normalizeNoteAttachmentFields(item); + normalizeNoteAttachmentFields(item, attachmentPlan.wireIdsFor(record)); } array.add(item); } return array; } - private static void normalizeNoteAttachmentFields(JsonObject note) throws IOException { + /** + * Replaces the local attachment fields with the wire references. + * + * @param wireIds the manifest id each of this record's logical attachment ids travels under; + * identical to the logical id except for a re-keyed collision. + */ + private static void normalizeNoteAttachmentFields( + JsonObject note, @NonNull Map wireIds) throws IOException { JsonArray manifestEntries = note.getAsJsonArray("attachmentsManifest"); JsonArray attachmentIds = new JsonArray(); JsonObject attachmentNames = new JsonObject(); + JsonObject aliases = new JsonObject(); if (manifestEntries != null) { for (JsonElement element : manifestEntries) { AttachmentManifestEntry attachment = AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - attachmentIds.add(attachment.id); + String wireId = wireIds.getOrDefault(attachment.id, attachment.id); + attachmentIds.add(wireId); + if (!wireId.equals(attachment.id)) { + aliases.addProperty(wireId, attachment.id); + } if (attachment.displayName != null && !attachment.displayName.isEmpty()) { - attachmentNames.addProperty(attachment.id, attachment.displayName); + attachmentNames.addProperty(wireId, attachment.displayName); } } } @@ -227,12 +249,16 @@ private static void normalizeNoteAttachmentFields(JsonObject note) throws IOExce // empty, and a decoded record then hashed differently from the local one that produced // it — a conflict against itself on every sync. note.remove("attachmentNames"); + note.remove(FIELD_ATTACHMENT_ID_ALIASES); if (attachmentIds.size() > 0) { note.add("attachmentIds", attachmentIds); } if (attachmentNames.size() > 0) { note.add("attachmentNames", attachmentNames); } + if (aliases.size() > 0) { + note.add(FIELD_ATTACHMENT_ID_ALIASES, aliases); + } } /** @@ -258,7 +284,8 @@ private static List dedupeAlternatives( } @NonNull - private static JsonArray alternativeArray(@NonNull List alternatives) + private static JsonArray alternativeArray( + @NonNull List alternatives, @NonNull AttachmentPlan attachmentPlan) throws IOException { JsonArray array = new JsonArray(); for (SyncRecord alternative : alternatives) { @@ -270,7 +297,7 @@ private static JsonArray alternativeArray(@NonNull List alternatives if (alternative.isTombstone()) { item.addProperty("deletedAt", alternative.getDeletedAt().toString()); } else if (alternative.getType() == SyncRecord.Type.NOTE) { - normalizeNoteAttachmentFields(item); + normalizeNoteAttachmentFields(item, attachmentPlan.wireIdsFor(alternative)); } array.add(item); } @@ -291,12 +318,23 @@ private static JsonArray tombstones(@NonNull SyncSnapshot snapshot) { return array; } + /** + * Lays out the bundle's attachment manifest for every version it carries. + * + *

The manifest is keyed by logical attachment id and a note references its entries by that + * id, so one id can describe only one blob per bundle. A live note and one of its unresolved + * alternatives may nonetheless carry different content under one id — the file was replaced in + * place, or two devices derived the same id — and refusing to encode that used to fail every + * publish for the account, before the conflict could even be stored for the user to settle. The + * later version's entry now travels under a derived id and the record carries the mapping back, + * so the decoded version is byte-for-byte the one that was published and keeps the identity + * every device's resolution bookkeeping refers to. + */ @NonNull - private static JsonArray collectAttachments( + private static AttachmentPlan planAttachments( @NonNull SyncSnapshot snapshot, @NonNull List alternatives) throws IOException { - JsonArray attachments = new JsonArray(); - Map seenById = new LinkedHashMap<>(); + AttachmentPlan plan = new AttachmentPlan(); Map seenByHash = new LinkedHashMap<>(); List notes = new ArrayList<>(snapshot.getLiveRecords(SyncRecord.Type.NOTE)); // An unresolved alternative is only recoverable if its blobs are described here too. @@ -311,11 +349,19 @@ private static JsonArray collectAttachments( for (JsonElement element : manifestEntries) { AttachmentManifestEntry attachment = AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - AttachmentManifestEntry sameId = seenById.putIfAbsent(attachment.id, attachment); + String wireId = attachment.id; + AttachmentManifestEntry sameId = plan.byWireId.get(wireId); if (sameId != null && !sameId.sameRemoteFile(attachment)) { - // The same logical attachment may appear in both a live note and one of its - // unresolved alternatives; only differing content is a contradiction. - throw new IOException("Two notes reference conflicting attachment metadata"); + wireId = aliasFor(attachment); + sameId = plan.byWireId.get(wireId); + if (sameId != null && !sameId.sameRemoteFile(attachment)) { + throw new IOException( + "Two notes reference conflicting attachment metadata"); + } + plan.wireIdsFor(record).put(attachment.id, wireId); + } + if (sameId == null) { + plan.byWireId.put(wireId, attachment.withId(wireId)); } AttachmentManifestEntry previous = seenByHash.putIfAbsent(attachment.sha256, attachment); @@ -327,10 +373,42 @@ private static JsonArray collectAttachments( if (seenByHash.size() > SyncBundleValidator.MAX_ATTACHMENT_COUNT) { throw new IOException("Sync bundle exceeds the schema-1 attachment limit"); } - for (AttachmentManifestEntry attachment : seenById.values()) { - attachments.add(attachment.toJson(false)); + return plan; + } + + /** Deterministic, so two devices publishing the same collision write the same bundle. */ + @NonNull + private static String aliasFor(@NonNull AttachmentManifestEntry attachment) { + return UUID.nameUUIDFromBytes( + ("attachment-alias\n" + attachment.id + "\n" + attachment.sha256) + .getBytes(StandardCharsets.UTF_8)) + .toString(); + } + + /** The manifest entries by wire id, and each record's logical-to-wire id mapping. */ + private static final class AttachmentPlan { + private final Map byWireId = new LinkedHashMap<>(); + private final Map> wireIdsByRecord = + new java.util.IdentityHashMap<>(); + + @NonNull + Map wireIdsFor(@NonNull SyncRecord record) { + Map wireIds = wireIdsByRecord.get(record); + if (wireIds == null) { + wireIds = new LinkedHashMap<>(); + wireIdsByRecord.put(record, wireIds); + } + return wireIds; + } + + @NonNull + JsonArray manifest() { + JsonArray attachments = new JsonArray(); + for (AttachmentManifestEntry attachment : byWireId.values()) { + attachments.add(attachment.toJson(false)); + } + return attachments; } - return attachments; } private static void parseLiveRecords( @@ -368,7 +446,15 @@ private static void hydrateNoteAttachments( throws IOException { JsonArray attachmentIds = payload.getAsJsonArray("attachmentIds"); JsonObject attachmentNames = payload.getAsJsonObject("attachmentNames"); - if (attachmentIds == null) return; + JsonObject aliases = payload.getAsJsonObject(FIELD_ATTACHMENT_ID_ALIASES); + // Wire-only, all three: the local store never produces them, and leaving one behind made + // a decoded record hash differently from the identical local one. + payload.remove("attachmentIds"); + payload.remove(FIELD_ATTACHMENT_ID_ALIASES); + if (attachmentIds == null) { + payload.remove("attachmentNames"); + return; + } JsonArray attachmentHashes = new JsonArray(); JsonArray manifest = new JsonArray(); // Keyed by logical attachment UUID, exactly as the wire carries it and exactly as @@ -377,14 +463,20 @@ private static void hydrateNoteAttachments( // attachment reported a conflict against itself on every sync, forever. JsonObject namesById = new JsonObject(); for (JsonElement element : attachmentIds) { - String attachmentId = element.getAsString(); - AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); + String wireId = element.getAsString(); + AttachmentManifestEntry attachment = attachmentsById.get(wireId); if (attachment == null) { throw new IOException("Note references an unknown attachment manifest entry"); } - JsonObject value = attachment.toJson(true); - if (attachmentNames != null && attachmentNames.has(attachmentId)) { - value.addProperty("displayName", attachmentNames.get(attachmentId).getAsString()); + // A re-keyed collision travels under a derived id; the record's own id is restored + // so the decoded version is the one that was published. + String attachmentId = + aliases != null && aliases.has(wireId) + ? aliases.get(wireId).getAsString() + : wireId; + JsonObject value = attachment.withId(attachmentId).toJson(true); + if (attachmentNames != null && attachmentNames.has(wireId)) { + value.addProperty("displayName", attachmentNames.get(wireId).getAsString()); } manifest.add(value); attachmentHashes.add(attachment.sha256); @@ -397,9 +489,6 @@ private static void hydrateNoteAttachments( // The display name restoreAttachments actually uses travels on the manifest entry above; // this map exists only so the payload matches the one the local store builds. payload.add("attachmentNames", namesById); - // Wire-only: the local store never produces it, and leaving it behind made a decoded - // record hash differently from the identical local one. - payload.remove("attachmentIds"); } @NonNull @@ -493,8 +582,8 @@ private static JsonArray requireArray(@NonNull JsonObject object, @NonNull Strin } @NonNull - private static String sha256(byte[] bytes) throws IOException { - return SyncBundleValidator.sha256(bytes); + private static String sha256(byte[] bytes) { + return Sha256.of(bytes); } public static final class DecodedBundle { @@ -601,11 +690,12 @@ public static AttachmentManifestEntry fromJson(@NonNull JsonObject value) if (!path.equals("attachments/" + sha256)) { throw new IOException("Sync bundle contains an invalid attachment path"); } + // Trimmed here, once, so every consumer sees the name the validator judged. String displayName = value.has("displayName") && !value.get("displayName").isJsonNull() && value.get("displayName").isJsonPrimitive() - ? value.get("displayName").getAsString() + ? value.get("displayName").getAsString().trim() : null; if (value.has("displayName") && !value.get("displayName").isJsonNull() @@ -635,5 +725,13 @@ JsonObject toJson(boolean includeDisplayName) { boolean sameRemoteFile(@NonNull AttachmentManifestEntry other) { return sha256.equals(other.sha256) && path.equals(other.path) && size == other.size; } + + /** The same blob under another logical id. */ + @NonNull + AttachmentManifestEntry withId(@NonNull String newId) { + return newId.equals(id) + ? this + : new AttachmentManifestEntry(newId, sha256, mimeType, size, path, displayName); + } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index 083ca977..6df66d36 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -10,13 +10,10 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.time.format.DateTimeParseException; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -178,6 +175,7 @@ private static void validateAttachmentReferences( throw new IOException("Sync note exceeds the attachment limit"); } JsonObject attachmentNames = note.getAsJsonObject("attachmentNames"); + validateAttachmentIdAliases(note, attachmentsById); for (JsonElement element : attachmentIds) { if (element == null || !element.isJsonPrimitive()) { throw new IOException("Sync note attachmentIds entry is invalid"); @@ -199,6 +197,33 @@ private static void validateAttachmentReferences( } } + /** + * Checks the map that lets two versions of one note carry different content under one logical + * attachment id; see {@code SyncBundleCodec.collectAttachments}. + */ + private static void validateAttachmentIdAliases( + @NonNull JsonObject note, + @NonNull Map attachmentsById) + throws IOException { + JsonElement aliases = note.get(SyncBundleCodec.FIELD_ATTACHMENT_ID_ALIASES); + if (aliases == null || aliases.isJsonNull()) { + return; + } + if (!aliases.isJsonObject()) { + throw new IOException("Sync note attachmentIdAliases is invalid"); + } + for (Map.Entry alias : aliases.getAsJsonObject().entrySet()) { + validateUuid(alias.getKey()); + if (!alias.getValue().isJsonPrimitive()) { + throw new IOException("Sync note attachmentIdAliases entry is invalid"); + } + validateUuid(alias.getValue().getAsString()); + if (!attachmentsById.containsKey(alias.getKey())) { + throw new IOException("Note aliases an unknown attachment manifest entry"); + } + } + } + /** * Validates the unresolved conflict versions a bundle carries. * @@ -511,23 +536,18 @@ private static void validateJsonValue(@NonNull JsonElement value, int depth) } } + /** + * Accepts a display name only in the form every consumer will see it: trimmed, then held to the + * one path-segment rule the attachment code shares. + * + *

Three validators used to exist with three answers for a name with trailing whitespace; the + * day any path built a file name from the display name, the bundle validator's answer and the + * URL parser's would have disagreed and the cleaner would have refused the note. + */ static void validateDisplayName(@NonNull String name) throws IOException { - String value = name.trim(); - if (value.isEmpty() - || value.equals(".") - || value.equals("..") - || value.length() > 255 - || value.contains("..") - || value.contains("/") - || value.contains("\\") - || new java.io.File(value).getName().equals(value) == false) { + if (!com.pasich.mynotes.extendedEditor.attach.AttachmentUrl.isSafeSegment(name.trim())) { throw new IOException("Sync attachment name is not a safe file name"); } - for (int i = 0; i < value.length(); i++) { - if (Character.isISOControl(value.charAt(i))) { - throw new IOException("Sync attachment name contains a control character"); - } - } } static void validateUuid(@NonNull String value) throws IOException { @@ -541,17 +561,8 @@ static void validateUuid(@NonNull String value) throws IOException { } @NonNull - static String sha256(@NonNull byte[] bytes) throws IOException { - try { - byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes); - StringBuilder value = new StringBuilder(digest.length * 2); - for (byte byteValue : digest) { - value.append(String.format(Locale.US, "%02x", byteValue & 0xff)); - } - return value.toString(); - } catch (NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } + static String sha256(@NonNull byte[] bytes) { + return Sha256.of(bytes); } /** Caps compressed input before ZIP parsing to make bundle-size limits independent of Drive. */ diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java index d3bc990b..c933046b 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java @@ -45,11 +45,18 @@ interface StableIdGenerator { String nextStableId(); } - /** Repoints a restored note's attachments when its row id had to change. */ + /** + * Moves a restored note's attachments to where its final row id expects them. + * + * @return true when the note's attachment or block JSON was rewritten and must be stored. + */ interface AttachmentRelocation { - void relocate(@NonNull Note note, int previousId); + boolean relocate(@NonNull Note note, int previousId); } + /** Keeps every {@code IN (...)} clause under SQLite's bound-variable limit. */ + private static final int QUERY_CHUNK = 500; + private final TransactionExecutor transactionExecutor; private final NoteDao noteDao; private final TaskDao taskDao; @@ -93,7 +100,9 @@ public T call() { (note, previousId) -> { com.pasich.mynotes.extendedEditor.attach.NoteAttachmentRelocator.Result moved = com.pasich.mynotes.extendedEditor.attach.NoteAttachmentRelocator - .relocate( + .adoptStaged( + com.pasich.mynotes.extendedEditor.attach + .AttachmentStorage.restoreStagingDir(context), com.pasich.mynotes.extendedEditor.attach .AttachmentStorage.baseDirPath(context), previousId, @@ -104,6 +113,7 @@ public T call() { note.setAttachments(moved.attachmentsJson); note.setValueJson(moved.valueJson); } + return moved.changed; }); } @@ -127,7 +137,7 @@ public T call() { syncMetadataDao, timeProvider, stableIdGenerator, - (note, previousId) -> {}); + (note, previousId) -> false); } SyncMutationCoordinator( @@ -175,10 +185,16 @@ public void insertTags(List incoming) { SyncMetadata.RECORD_TYPE_TAG, extractTagIds(tags)); // Same REPLACE-insert collision as notes; see insertNotes. + Set taken = new LinkedHashSet<>(); + for (Tag existing : tagsByIds(extractTagIds(tags))) { + taken.add(existing.getId()); + } List keepingId = new ArrayList<>(); List reassigned = new ArrayList<>(); + Set claimed = new LinkedHashSet<>(); for (Tag tag : tags) { - if (tag.getId() > 0 && tagsDao.getTagSync(tag.getId()) != null) { + if (tag.getId() > 0 + && (taken.contains(tag.getId()) || !claimed.add(tag.getId()))) { tag.id = 0; reassigned.add(tag); } else { @@ -284,7 +300,11 @@ public void insertNotes(List incoming) { if (incoming == null || incoming.isEmpty()) return; transactionExecutor.run( () -> { - List notes = withoutNotesAlreadyPresent(incoming); + // One query for every incoming id rather than one per note per pass: a + // large restore ran four point lookups per note inside the transaction and + // sat on the restore dialog for tens of seconds doing nothing else. + Map existingById = notesByIds(extractNoteIds(incoming)); + List notes = withoutNotesAlreadyPresent(incoming, existingById); if (notes.isEmpty()) return null; long timestamp = resolveBatchTimestamp( @@ -294,13 +314,18 @@ public void insertNotes(List incoming) { // their id first leaves the autoincrement counter past all of them, which is // what stops a reassigned note being handed an id a later note in the same // batch is about to claim: addNotes is a REPLACE insert, so that collision - // silently destroyed one of the two restored notes. + // silently destroyed one of the two restored notes. An id claimed twice + // within the batch itself is the same collision: the second note would + // REPLACE the first, so only the first may keep it. List keepingId = new ArrayList<>(); List reassigned = new ArrayList<>(); Map previousIds = new java.util.IdentityHashMap<>(); + Set claimed = new LinkedHashSet<>(); for (Note note : notes) { previousIds.put(note, note.getId()); - if (note.getId() > 0 && noteDao.getNoteSync(note.getId()) != null) { + if (note.getId() > 0 + && (existingById.containsKey(note.getId()) + || !claimed.add(note.getId()))) { note.setId(0); reassigned.add(note); } else { @@ -324,10 +349,9 @@ private void assignInsertedNoteIds( int previous = previousIds.get(note); int localId = resolveIntId(note.getId(), insertedIds[i]); note.setId(localId); - if (previous > 0 && previous != localId) { - // Its attachments were extracted under the old id and would otherwise share a - // folder with whichever note owns that id now. - attachmentRelocation.relocate(note, previous); + // Its attachments were staged under the id the archive knew; they move into the + // folder of the id it has now, whether or not the two differ. + if (previous > 0 && attachmentRelocation.relocate(note, previous)) { noteDao.updateNoteContent( localId, note.getTitle(), @@ -629,10 +653,11 @@ private long insertNoteInternal(@NonNull Note note, long timestamp) { * the existing one instead of overwriting it. */ @NonNull - private List withoutNotesAlreadyPresent(@NonNull List incoming) { + private static List withoutNotesAlreadyPresent( + @NonNull List incoming, @NonNull Map existingById) { List result = new ArrayList<>(incoming.size()); for (Note note : incoming) { - Note existing = note.getId() > 0 ? noteDao.getNoteSync(note.getId()) : null; + Note existing = note.getId() > 0 ? existingById.get(note.getId()) : null; if (existing == null || !isSameNoteContent(existing, note)) { result.add(note); } @@ -640,6 +665,44 @@ private List withoutNotesAlreadyPresent(@NonNull List incoming) { return result; } + /** The existing rows for these ids, fetched in bounded batches. */ + @NonNull + private Map notesByIds(@NonNull List ids) { + Map result = new java.util.HashMap<>(); + List positive = new ArrayList<>(); + for (Long id : ids) { + if (id != null && id > 0L) positive.add(id.intValue()); + } + for (List chunk : chunks(positive)) { + for (Note note : noteDao.getNotesByIdsSync(chunk)) { + result.put(note.getId(), note); + } + } + return result; + } + + @NonNull + private List tagsByIds(@NonNull List ids) { + List result = new ArrayList<>(); + List positive = new ArrayList<>(); + for (Long id : ids) { + if (id != null && id > 0L) positive.add(id); + } + for (List chunk : chunks(positive)) { + result.addAll(tagsDao.getTagsByIdsSync(chunk)); + } + return result; + } + + @NonNull + private static List> chunks(@NonNull List values) { + List> result = new ArrayList<>(); + for (int start = 0; start < values.size(); start += QUERY_CHUNK) { + result.add(values.subList(start, Math.min(values.size(), start + QUERY_CHUNK))); + } + return result; + } + /** True when two rows carry the same user-visible note. */ private static boolean isSameNoteContent(@NonNull Note existing, @NonNull Note incoming) { return equalText(existing.getTitle(), incoming.getTitle()) @@ -704,7 +767,7 @@ private void markDeletedRecord(@NonNull String recordType, long localId, long ti } private void ensureMetadataRow(@NonNull String recordType, long localId) { - if (syncMetadataDao.exists(recordType, localId)) return; + // INSERT OR IGNORE: a row that exists is left alone without a lookup first. syncMetadataDao.insertIfAbsent( new SyncMetadataEntity( recordType, localId, stableIdGenerator.nextStableId(), 0L, null)); @@ -725,11 +788,15 @@ private long resolveBatchTimestamp(@NonNull String recordType, @NonNull List localIds) { + List positive = new ArrayList<>(); for (Long localId : localIds) { - if (localId == null || localId <= 0L) continue; - if (!syncMetadataDao.exists(recordType, localId)) return true; + if (localId != null && localId > 0L) positive.add(localId); + } + Set known = new LinkedHashSet<>(); + for (List chunk : chunks(positive)) { + known.addAll(syncMetadataDao.getExistingLocalIds(recordType, chunk)); } - return false; + return !known.containsAll(positive); } private static List extractNoteIds(List notes) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java index 9ecbfdb0..c0ed704e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java @@ -8,9 +8,6 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -80,7 +77,41 @@ private SyncRecord( throw new IllegalArgumentException("deletedAt must not be before updatedAt"); } this.deletedAt = deletedAt; - this.payload = Objects.requireNonNull(payload, "payload").deepCopy(); + this.payload = normalize(type, Objects.requireNonNull(payload, "payload").deepCopy()); + } + + /** + * Puts a payload into the one shape its version hash is taken from. + * + *

A note's attachment fields are present exactly when it has attachments. The local build, + * the bundle encoder and the bundle decoder each used to enforce that separately, and every + * time one of them drifted — an empty {@code attachmentNames} left on the wire, three empty + * arrays emitted for a note with none — a decoded record hashed differently from the identical + * local one and every affected note conflicted with itself on every sync. Every record passes + * through here, whichever side built it, so the three sites can no longer disagree. + */ + @NonNull + private static JsonObject normalize(@NonNull Type type, @NonNull JsonObject payload) { + if (type != Type.NOTE) { + return payload; + } + for (String field : new String[] {"attachmentsManifest", "attachmentHashes"}) { + JsonElement value = payload.get(field); + if (value != null + && (value.isJsonNull() + || !value.isJsonArray() + || value.getAsJsonArray().size() == 0)) { + payload.remove(field); + } + } + JsonElement names = payload.get("attachmentNames"); + if (names != null + && (names.isJsonNull() + || !names.isJsonObject() + || names.getAsJsonObject().size() == 0)) { + payload.remove("attachmentNames"); + } + return payload; } @NonNull @@ -171,18 +202,7 @@ private static void validateCanonicalUuid(String id) { @NonNull private static String sha256(String value) { - try { - byte[] digest = - MessageDigest.getInstance("SHA-256") - .digest(value.getBytes(StandardCharsets.UTF_8)); - StringBuilder hex = new StringBuilder(digest.length * 2); - for (byte byteValue : digest) { - hex.append(String.format("%02x", byteValue & 0xff)); - } - return hex.toString(); - } catch (NoSuchAlgorithmException exception) { - throw new IllegalStateException("SHA-256 is unavailable", exception); - } + return Sha256.of(value); } @NonNull diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 254a4ade..16e59be9 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -5,11 +5,8 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Clock; import java.time.Instant; import java.util.Collection; @@ -63,6 +60,22 @@ public SyncService(@NonNull SyncStore store, @NonNull SyncMerger merger, @NonNul private static final long LOCK_WAIT_SECONDS = 5L; + /** + * Runs {@code action} while no sync is in flight, waiting for a running one to finish first. + * + *

For work that tears down what a sync writes — the disconnect wipe of state, conflicts and + * cached blobs. Done concurrently, that wipe raced the six-hourly worker: the worker's cache + * directory vanished under it and its final state and conflict rows landed after the wipe. + */ + public static void runWhileNoSyncRuns(@NonNull Runnable action) { + SYNC_LOCK.lock(); + try { + action.run(); + } finally { + SYNC_LOCK.unlock(); + } + } + /** Runs one serialized manual synchronization attempt and returns its durable final state. */ @NonNull public SyncState sync(@NonNull SyncBackend backend) { @@ -152,7 +165,8 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // The merged snapshot contains only the deterministic winner. A conflict row is not // durable unless the loser can later be restored as well, so preflight and pin each // version independently; SyncSnapshot deliberately forbids two versions of one ID. - synchronizeAttachments(backend, merged, expectedSizes); + java.util.Set synchronizedHashes = new java.util.HashSet<>(); + synchronizeAttachments(backend, merged, expectedSizes, synchronizedHashes); for (SyncMergeResult.Conflict conflict : allConflicts) { // Best effort. The merged snapshot's own blobs are mandatory and were just // transferred above; these are the extra copies that let a conflict be resolved @@ -161,8 +175,8 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // syncing anything at all, including the devices that could never resolve it. // Resolution still verifies before it applies, so a version that cannot be // materialized simply cannot be chosen. - pinConflictVersionQuietly(backend, conflict.getWinner()); - pinConflictVersionQuietly(backend, conflict.getLoser()); + pinConflictVersionQuietly(backend, conflict.getWinner(), synchronizedHashes); + pinConflictVersionQuietly(backend, conflict.getLoser(), synchronizedHashes); } if (needsPublication( @@ -341,13 +355,27 @@ private static boolean snapshotsMatch( return true; } + /** + * Makes every blob {@code merged} references available, verified, at both endpoints. + * + * @param synchronizedHashes blobs already settled by an earlier call this sync, skipped here + * and extended with the ones settled now. A conflict version shares most of its blobs with + * the merged snapshot, and re-verifying each one meant re-hashing and re-downloading it + * once per version while the conflict stayed open. + */ private void synchronizeAttachments( - SyncBackend backend, SyncSnapshot merged, Map expectedSizes) + SyncBackend backend, + SyncSnapshot merged, + Map expectedSizes, + java.util.Set synchronizedHashes) throws IOException { Collection hashes = Objects.requireNonNull(store.getAttachmentHashes(merged), "hashes"); for (String hash : hashes) { validateHash(hash); + if (!synchronizedHashes.add(hash)) { + continue; + } if (store.hasAttachment(hash)) { Long expectedSize = expectedSizes.get(hash); // Index lookup only; the bytes are checked once, below. @@ -405,9 +433,11 @@ private void synchronizeAttachments( /** Pins a conflict version's blobs, logging rather than failing the whole sync. */ private void pinConflictVersionQuietly( - @NonNull SyncBackend backend, @NonNull SyncRecord record) { + @NonNull SyncBackend backend, + @NonNull SyncRecord record, + @NonNull java.util.Set synchronizedHashes) { try { - pinConflictVersion(backend, record); + pinConflictVersion(backend, record, synchronizedHashes); } catch (IOException unavailable) { Log.w( TAG, @@ -416,15 +446,26 @@ private void pinConflictVersionQuietly( } } - /** Pins required conflict blobs into the store's durable content-addressed cache. */ - private void pinConflictVersion(@NonNull SyncBackend backend, @NonNull SyncRecord record) + /** + * Pins required conflict blobs into the store's durable content-addressed cache. + * + *

A blob already in that cache is left alone: copying it onto itself rewrote hundreds of + * megabytes per sync for as long as a conflict on a large note stayed open. + */ + private void pinConflictVersion( + @NonNull SyncBackend backend, + @NonNull SyncRecord record, + @NonNull java.util.Set synchronizedHashes) throws IOException { if (record.isTombstone()) return; SyncSnapshot snapshot = new SyncSnapshot(java.util.Collections.singletonList(record)); Map expectedSizes = attachmentSizes(snapshot); - synchronizeAttachments(backend, snapshot, expectedSizes); + synchronizeAttachments(backend, snapshot, expectedSizes, synchronizedHashes); for (String hash : store.getAttachmentHashes(snapshot)) { Long expectedSize = expectedSizes.get(hash); + if (store.hasDurableAttachment(hash, expectedSize == null ? -1L : expectedSize)) { + continue; + } copyVerified(hash, expectedSize, store.readAttachment(hash), store::writeAttachment); } } @@ -434,14 +475,9 @@ private void verifyAttachment(String hash, Long expectedSize, InputStream source if (source == null) { throw new IOException("Required attachment is unavailable: " + hash); } - try (InputStream input = source; - VerifyingInputStream verified = - new VerifyingInputStream(input, hash, expectedSize)) { - byte[] buffer = new byte[8192]; - while (verified.read(buffer) != -1) { - // Consume the complete blob before accepting an existing remote attachment. - } - verified.verifyEndOfStream(); + // Consume the complete blob before accepting an existing attachment. + try (InputStream input = source) { + VerifyingInputStream.verify(input, hash, expectedSize); } } @@ -538,126 +574,4 @@ private static void validateHash(String hash) throws IOException { private interface AttachmentWriter { void write(String hash, long sizeBytes, InputStream content) throws IOException; } - - /** Verifies the hash only after the receiving endpoint consumed every byte. */ - private static final class VerifyingInputStream extends FilterInputStream { - private final MessageDigest digest; - private final String expectedHash; - private final Long expectedSize; - private long byteCount; - private VerificationState verificationState = VerificationState.UNVERIFIED; - private AttachmentIntegrityException integrityFailure; - - private enum VerificationState { - UNVERIFIED, - VERIFIED, - FAILED - } - - VerifyingInputStream(InputStream input, String expectedHash, Long expectedSize) { - super(input); - this.expectedHash = expectedHash; - this.expectedSize = expectedSize; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException exception) { - throw new IllegalStateException("SHA-256 is unavailable", exception); - } - } - - @Override - public int read() throws IOException { - rethrowIntegrityFailure(); - int value = super.read(); - if (value >= 0) { - digest.update((byte) value); - byteCount++; - enforceSizeLimit(); - } else { - verifyEndOfStream(); - } - return value; - } - - @Override - public int read(byte[] buffer, int offset, int length) throws IOException { - rethrowIntegrityFailure(); - int read = super.read(buffer, offset, length); - if (read > 0) { - digest.update(buffer, offset, read); - byteCount += read; - enforceSizeLimit(); - } else if (read < 0) { - verifyEndOfStream(); - } - return read; - } - - private void enforceSizeLimit() throws IOException { - if (byteCount > SyncBundleValidator.MAX_ATTACHMENT_BYTES) { - failIntegrity("Attachment exceeds the sync size limit"); - } - } - - /** - * Checks the digest, draining anything the destination left behind first. - * - *

Reached from {@link #read} at end of stream, so a destination that streams straight to - * its final location still learns about a mismatch before it commits. - */ - void verifyEndOfStream() throws IOException { - if (verificationState == VerificationState.VERIFIED) { - return; - } - rethrowIntegrityFailure(); - try { - drainRemaining(); - String actualHash = toHex(digest.digest()); - if (!expectedHash.equals(actualHash)) { - failIntegrity("Attachment checksum does not match its declared hash"); - } - if (expectedSize != null && expectedSize.longValue() != byteCount) { - failIntegrity("Attachment size does not match its declared size"); - } - verificationState = VerificationState.VERIFIED; - } catch (AttachmentIntegrityException failure) { - integrityFailure = failure; - verificationState = VerificationState.FAILED; - throw failure; - } - } - - private void failIntegrity(String message) throws AttachmentIntegrityException { - AttachmentIntegrityException failure = new AttachmentIntegrityException(message); - integrityFailure = failure; - verificationState = VerificationState.FAILED; - throw failure; - } - - private void rethrowIntegrityFailure() throws AttachmentIntegrityException { - if (verificationState == VerificationState.FAILED && integrityFailure != null) { - throw integrityFailure; - } - } - - /** Reads through {@code super} so the digest covers bytes the destination skipped. */ - private void drainRemaining() throws IOException { - byte[] scratch = new byte[8192]; - int read; - while ((read = super.read(scratch, 0, scratch.length)) != -1) { - digest.update(scratch, 0, read); - byteCount += read; - enforceSizeLimit(); - } - } - - @NonNull - private static String toHex(byte[] bytes) { - StringBuilder result = new StringBuilder(bytes.length * 2); - for (byte value : bytes) { - result.append(String.format("%02x", value & 0xff)); - } - return result.toString(); - } - } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java index 8a0bd8db..48dac670 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java @@ -71,18 +71,29 @@ default java.util.Set getResolvedAlternativeIds() throws IOException { /** True when the complete attachment is locally available. */ boolean hasAttachment(@NonNull String sha256) throws IOException; + /** + * True when the blob already sits in the store's own durable cache, as written by {@link + * #writeAttachment}, rather than only in a note's folder that a later edit may empty. + * + *

Lets the service pin a conflict version's blobs without copying a cached blob onto itself; + * a store without such a cache answers false and is simply written to again. + * + * @param sizeBytes the declared size, or a negative value to accept any. + */ + default boolean hasDurableAttachment(@NonNull String sha256, long sizeBytes) + throws IOException { + return false; + } + /** Opens one complete local attachment. The caller closes the returned stream. */ @NonNull InputStream readAttachment(@NonNull String sha256) throws IOException; /** - * Stores one complete attachment locally. + * Stores one immutable blob, streaming it rather than holding it in memory. * *

The implementation must consume the stream before returning and must not expose a partial * file after an exception. - */ - /** - * Stores one immutable blob, streaming it rather than holding it in memory. * * @param sizeBytes the blob's declared size, or a negative value when it is unknown. A known * size lets an implementation avoid buffering the whole blob to compute a content length. diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java b/app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java new file mode 100644 index 00000000..bf9cebfa --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java @@ -0,0 +1,147 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; + +/** + * Checks a content-addressed blob against its declared hash and size as the bytes go past. + * + *

This is the single verifier for every attachment transfer: local file to Drive, Drive to the + * local cache, cache to a note folder, and the Drive-to-Drive copy into the canonical root. Four + * separate implementations used to exist with four slightly different rules — one enforced the size + * ceiling, one did not, one returned false where the others threw — so a blob accepted on one path + * could be rejected on the next and leave a bundle pointing at bytes no device would serve. + * + *

Verification happens at end of stream, which is reached inside the destination's own read + * loop, so a destination that streams straight to its final location still learns about a mismatch + * before it commits. {@link #verifyEndOfStream()} drains whatever the destination left unread, so a + * consumer that stopped early cannot accept a blob it never finished checking. + */ +final class VerifyingInputStream extends FilterInputStream { + + private final MessageDigest digest = Sha256.newDigest(); + private final String expectedHash; + private final Long expectedSize; + private final long maxBytes; + private long byteCount; + private boolean verified; + private AttachmentIntegrityException integrityFailure; + + VerifyingInputStream( + @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) { + this(input, expectedHash, expectedSize, SyncBundleValidator.MAX_ATTACHMENT_BYTES); + } + + VerifyingInputStream( + @NonNull InputStream input, + @NonNull String expectedHash, + @Nullable Long expectedSize, + long maxBytes) { + super(input); + this.expectedHash = expectedHash; + this.expectedSize = expectedSize; + this.maxBytes = maxBytes; + } + + /** + * Reads {@code input} to its end and verifies it; the caller closes the stream. + * + * @return the blob's size in bytes. + */ + static long verify( + @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) + throws IOException { + VerifyingInputStream verifying = + new VerifyingInputStream(input, expectedHash, expectedSize); + verifying.verifyEndOfStream(); + return verifying.bytesRead(); + } + + /** Bytes seen so far; the blob's size once {@link #verifyEndOfStream()} has passed. */ + long bytesRead() { + return byteCount; + } + + @Override + public int read() throws IOException { + rethrowIntegrityFailure(); + int value = super.read(); + if (value >= 0) { + digest.update((byte) value); + byteCount++; + enforceSizeLimit(); + } else { + verifyEndOfStream(); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + rethrowIntegrityFailure(); + int read = super.read(buffer, offset, length); + if (read > 0) { + digest.update(buffer, offset, read); + byteCount += read; + enforceSizeLimit(); + } else if (read < 0) { + verifyEndOfStream(); + } + return read; + } + + private void enforceSizeLimit() throws AttachmentIntegrityException { + if (byteCount > maxBytes) { + failIntegrity("Attachment exceeds the sync size limit"); + } + } + + /** + * Checks the digest, draining anything the destination left behind first. + * + *

Idempotent: the destination's read loop reaches end of stream and verifies, and the caller + * verifies again after the destination returns; the second call is a no-op. + */ + void verifyEndOfStream() throws IOException { + if (verified) { + return; + } + rethrowIntegrityFailure(); + drainRemaining(); + String actualHash = Sha256.hex(digest.digest()); + if (!expectedHash.equals(actualHash)) { + failIntegrity("Attachment checksum does not match its declared hash"); + } + if (expectedSize != null && expectedSize.longValue() != byteCount) { + failIntegrity("Attachment size does not match its declared size"); + } + verified = true; + } + + private void failIntegrity(String message) throws AttachmentIntegrityException { + AttachmentIntegrityException failure = new AttachmentIntegrityException(message); + integrityFailure = failure; + throw failure; + } + + private void rethrowIntegrityFailure() throws AttachmentIntegrityException { + if (integrityFailure != null) { + throw integrityFailure; + } + } + + /** Reads through {@code super} so the digest covers bytes the destination skipped. */ + private void drainRemaining() throws IOException { + byte[] scratch = new byte[8192]; + int read; + while ((read = super.read(scratch, 0, scratch.length)) != -1) { + digest.update(scratch, 0, read); + byteCount += read; + enforceSizeLimit(); + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java index ec814498..a34fb3a4 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java @@ -205,6 +205,19 @@ public static File baseDirPath(Context ctx) { return new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR); } + /** + * Where a backup's attachments are unpacked before any note has been inserted. + * + *

Extracting straight into {@link #baseDirPath} overwrote the files of whatever local note + * happened to share a row id with a note in the archive, before the restore had decided whether + * that note would even keep its id. Files wait here until the row exists, then move into the + * folder of the id the note actually received. Under the cache directory so a restore that + * never finishes leaves only something the system may reclaim. + */ + public static File restoreStagingDir(Context ctx) { + return new File(new File(ctx.getCacheDir(), "restore-staging"), ATTACHMENTS_BASE_DIR); + } + /** * Builds the canonical URL for a file this app just wrote into a note's folder. * diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java index 9a23ec65..734f4105 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java @@ -149,8 +149,14 @@ private static int indexOfAny(@NonNull String value, char first, char second) { return -1; } - /** True only for a name that is exactly one ordinary path segment. */ - static boolean isSafeSegment(@Nullable String name) { + /** + * True only for a name that is exactly one ordinary path segment. + * + *

The single rule for attachment names: the sync store and the bundle validator both defer + * to it, so a name one of them accepted can never be one the other refuses to build a path + * from. + */ + public static boolean isSafeSegment(@Nullable String name) { if (name == null) { return false; } diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java new file mode 100644 index 00000000..8af4cd84 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java @@ -0,0 +1,146 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.ArrayList; +import java.util.List; + +/** + * The one walk over an Editor.js document that knows where attachment references live. + * + *

A block carries its file either as {@code data.file} (the attaches and image tools) or as + * {@code data.files[]}; the block type is deliberately not consulted, because a reference is a + * reference whichever tool wrote it. Three separate walkers used to exist — one filtered by type, + * one ignored {@code files[]}, one did both — so the same note could relocate correctly after a ZIP + * restore and render broken links after a Drive restore. + * + *

Deliberately free of {@code android.*}: sync hashes the rewritten document, so the rewrite has + * to be provably deterministic under ordinary JVM tests. + */ +public final class EditorAttachmentBlocks { + + /** Maps one stored URL to its replacement, or returns {@code null} to leave it alone. */ + public interface UrlMapper { + @Nullable + String map(@NonNull String url); + } + + private EditorAttachmentBlocks() {} + + /** + * Rewrites every file URL the mapper has a replacement for. + * + * @return the rewritten document, or {@code valueJson} itself when nothing changed or the + * document could not be read. Returning the original string verbatim, rather than a + * re-serialization of it, keeps an untouched note byte-identical on every device. + */ + @Nullable + public static String rewriteUrls(@Nullable String valueJson, @NonNull UrlMapper mapper) { + if (valueJson == null || valueJson.trim().isEmpty()) { + return valueJson; + } + JsonArray blocks; + try { + blocks = JsonParser.parseString(valueJson).getAsJsonArray(); + } catch (RuntimeException unreadable) { + return valueJson; + } + boolean changed = false; + for (JsonObject file : fileObjects(blocks)) { + String url = file.get("url").getAsString(); + String replacement = mapper.map(url); + if (replacement != null && !replacement.equals(url)) { + file.addProperty("url", replacement); + changed = true; + } + } + // JsonElement.toString(), not Gson.toJson(): the latter HTML-escapes text, and both + // directions of a sync must serialize identically or the hashes disagree. + return changed ? blocks.toString() : valueJson; + } + + /** Every file URL in document order. */ + @NonNull + public static List fileUrls(@Nullable String valueJson) { + List urls = new ArrayList<>(); + if (valueJson == null || valueJson.trim().isEmpty()) { + return urls; + } + try { + for (JsonObject file : + fileObjects(JsonParser.parseString(valueJson).getAsJsonArray())) { + urls.add(file.get("url").getAsString()); + } + } catch (RuntimeException unreadable) { + return new ArrayList<>(); + } + return urls; + } + + /** The file object of the block with {@code blockId}, or {@code null}. */ + @Nullable + public static JsonObject findFile(@Nullable String valueJson, @Nullable String blockId) { + if (valueJson == null || valueJson.isEmpty() || blockId == null || blockId.isEmpty()) { + return null; + } + try { + for (JsonElement element : JsonParser.parseString(valueJson).getAsJsonArray()) { + if (!element.isJsonObject()) continue; + JsonObject block = element.getAsJsonObject(); + if (!block.has("id") || !blockId.equals(block.get("id").getAsString())) continue; + List files = fileObjectsOf(block); + return files.isEmpty() ? null : files.get(0); + } + } catch (RuntimeException unreadable) { + return null; + } + return null; + } + + @NonNull + private static List fileObjects(@NonNull JsonArray blocks) { + List files = new ArrayList<>(); + for (JsonElement element : blocks) { + if (element.isJsonObject()) { + files.addAll(fileObjectsOf(element.getAsJsonObject())); + } + } + return files; + } + + /** {@code data.file} first, then {@code data.files[]}, each only when it carries a URL. */ + @NonNull + private static List fileObjectsOf(@NonNull JsonObject block) { + List files = new ArrayList<>(); + JsonElement dataElement = block.get("data"); + if (dataElement == null || !dataElement.isJsonObject()) { + return files; + } + JsonObject data = dataElement.getAsJsonObject(); + JsonElement file = data.get("file"); + if (hasUrl(file)) { + files.add(file.getAsJsonObject()); + } + JsonElement list = data.get("files"); + if (list != null && list.isJsonArray()) { + for (JsonElement candidate : list.getAsJsonArray()) { + if (hasUrl(candidate)) { + files.add(candidate.getAsJsonObject()); + } + } + } + return files; + } + + private static boolean hasUrl(@Nullable JsonElement element) { + if (element == null || !element.isJsonObject()) { + return false; + } + JsonElement url = element.getAsJsonObject().get("url"); + return url != null && url.isJsonPrimitive() && url.getAsJsonPrimitive().isString(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java index 463270b0..de9a5495 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java @@ -40,8 +40,15 @@ public static final class Result { private NoteAttachmentRelocator() {} + /** Turns one stored reference into its new URL, or {@code null} to leave it alone. */ + private interface ReferenceMover { + @Nullable + String move(@NonNull String url); + } + /** - * Repoints every reference from {@code previousNoteId} to {@code newNoteId}. + * Repoints every reference from {@code previousNoteId} to {@code newNoteId}, copying the files + * out of the old note's live folder. * * @param attachmentsRoot the app-private {@code attachments} directory. * @param attachmentsJson the note's attachments column. @@ -57,6 +64,63 @@ public static Result relocate( if (previousNoteId <= 0 || newNoteId <= 0 || previousNoteId == newNoteId) { return new Result(attachmentsJson, valueJson, false); } + return rewrite( + attachmentsJson, + valueJson, + url -> moveReference(attachmentsRoot, previousNoteId, newNoteId, url)); + } + + /** + * Moves a restored note's staged attachments into the folder of the id it actually received. + * + *

A backup's files are unpacked into a staging directory rather than the live folders, so + * this runs for every restored note, whether or not its id changed. A file already present + * under the same name in the destination is reused when it holds the same bytes and left alone + * otherwise — the staged copy then gets a fresh name — so a restore can never overwrite a file + * another note still shows. A reference with nothing staged falls back to the live-folder + * relocation above, which is what a note restored by an older release relied on. + * + * @param stagingRoot the staging directory that stands in for {@code attachments}. + * @param attachmentsRoot the app-private {@code attachments} directory. + */ + @NonNull + public static Result adoptStaged( + @NonNull File stagingRoot, + @NonNull File attachmentsRoot, + int previousNoteId, + int newNoteId, + @Nullable String attachmentsJson, + @Nullable String valueJson) { + if (previousNoteId <= 0 || newNoteId <= 0) { + return new Result(attachmentsJson, valueJson, false); + } + File stagedFolder = new File(stagingRoot, "note_" + previousNoteId); + Result result = + rewrite( + attachmentsJson, + valueJson, + url -> { + String adopted = + adoptReference( + stagedFolder, + attachmentsRoot, + previousNoteId, + newNoteId, + url); + if (adopted != null || previousNoteId == newNoteId) { + return adopted; + } + return moveReference(attachmentsRoot, previousNoteId, newNoteId, url); + }); + // Whatever the note did not reference was never going to be shown; it need not linger. + deleteRecursively(stagedFolder); + return result; + } + + /** Rewrites the column first and, only when it changed, the blocks that mirror it. */ + @NonNull + private static Result rewrite( + @Nullable String attachmentsJson, @Nullable String valueJson, ReferenceMover mover) { String movedAttachments = attachmentsJson; boolean changed = false; @@ -67,13 +131,10 @@ public static Result relocate( if (!element.isJsonObject()) continue; JsonObject entry = element.getAsJsonObject(); if (!entry.has("url") || !entry.get("url").isJsonPrimitive()) continue; - String rewritten = - moveReference( - attachmentsRoot, - previousNoteId, - newNoteId, - entry.get("url").getAsString()); - if (rewritten != null) { + String url = entry.get("url").getAsString(); + String rewritten = mover.move(url); + // A staged file adopted under the very same URL changes nothing worth storing. + if (rewritten != null && !rewritten.equals(url)) { entry.addProperty("url", rewritten); changed = true; } @@ -88,41 +149,82 @@ public static Result relocate( } String movedValueJson = valueJson; - if (changed && valueJson != null && !valueJson.trim().isEmpty()) { - try { - JsonArray blocks = JsonParser.parseString(valueJson).getAsJsonArray(); - boolean rewroteBlock = false; - for (JsonElement element : blocks) { - if (!element.isJsonObject()) continue; - JsonObject data = element.getAsJsonObject().getAsJsonObject("data"); - if (data == null) continue; - JsonObject file = data.getAsJsonObject("file"); - if (file == null || !file.has("url") || !file.get("url").isJsonPrimitive()) { - continue; - } - String rewritten = - moveReference( - attachmentsRoot, - previousNoteId, - newNoteId, - file.get("url").getAsString()); - if (rewritten != null) { - file.addProperty("url", rewritten); - rewroteBlock = true; - } - } - if (rewroteBlock) { - movedValueJson = blocks.toString(); - } - } catch (RuntimeException unreadable) { - // Keep the blocks untouched rather than risk corrupting the note's content. - movedValueJson = valueJson; - } + if (changed) { + // The blocks are what the editor renders; the shared walker leaves an unreadable + // document untouched rather than risk corrupting the note's content. + movedValueJson = EditorAttachmentBlocks.rewriteUrls(valueJson, mover::move); } return new Result(movedAttachments, movedValueJson, changed); } + /** + * Copies one staged file into the new note's folder and returns its new URL. + * + * @return the rewritten URL, or {@code null} when the reference does not belong to the old note + * or nothing was staged for it. + */ + @Nullable + private static String adoptReference( + @NonNull File stagedFolder, + @NonNull File attachmentsRoot, + int previousNoteId, + int newNoteId, + @NonNull String url) { + AttachmentUrl parsed = AttachmentUrl.parse(url); + if (parsed == null || !parsed.getNoteFolder().equals("note_" + previousNoteId)) { + return null; + } + File source = new File(stagedFolder, parsed.getFileName()); + if (!source.isFile()) { + return null; + } + File targetFolder = new File(attachmentsRoot, "note_" + newNoteId); + try { + if (!targetFolder.isDirectory() && !targetFolder.mkdirs()) { + return null; + } + File target = new File(targetFolder, parsed.getFileName()); + if (target.exists() && !sameContent(source, target)) { + target = new File(targetFolder, uniqueName(parsed.getFileName())); + } + if (!target.exists()) { + Files.copy(source.toPath(), target.toPath()); + } + return AttachmentUrl.canonical(newNoteId, target.getName()); + } catch (IOException | SecurityException | IllegalArgumentException failure) { + return null; + } + } + + private static boolean sameContent(@NonNull File first, @NonNull File second) + throws IOException { + return first.length() == second.length() + && java.util.Arrays.equals( + Files.readAllBytes(first.toPath()), Files.readAllBytes(second.toPath())); + } + + /** {@code name.ext} becomes {@code name-.ext}, still a safe single segment. */ + @NonNull + private static String uniqueName(@NonNull String fileName) { + String suffix = java.util.UUID.randomUUID().toString(); + int dot = fileName.lastIndexOf('.'); + if (dot <= 0) { + return fileName + "-" + suffix; + } + return fileName.substring(0, dot) + "-" + suffix + fileName.substring(dot); + } + + private static void deleteRecursively(@NonNull File file) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); + } + /** * Copies one referenced file into the new note's folder and returns its new URL. * diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java index 34007226..4f5d604f 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java @@ -17,39 +17,16 @@ public class EditorJsonUtils { * data.file - data.files[] */ public static EditorAttachment findAttachmentByBlockId(Note note, String blockId) { + if (note == null) return null; try { - if (note == null || blockId == null || blockId.isEmpty()) return null; - - String json = note.getValueJson(); - if (json == null || json.isEmpty()) return null; - - JSONArray blocks = new JSONArray(json); - - for (int i = 0; i < blocks.length(); i++) { - JSONObject block = blocks.optJSONObject(i); - if (block == null) continue; - - if (!blockId.equals(block.optString("id"))) continue; - - JSONObject data = block.optJSONObject("data"); - if (data == null) return null; - - // data.file - JSONObject fileObj = data.optJSONObject("file"); - if (fileObj != null) { - return EditorAttachment.fromJsonObject(fileObj); - } - - // data.files[] - JSONArray filesArr = data.optJSONArray("files"); - if (filesArr != null && filesArr.length() > 0) { - JSONObject f = filesArr.optJSONObject(0); - if (f != null) return EditorAttachment.fromJsonObject(f); - } - - return null; - } - + // The same walk sync and restore use, so a block one of them can see is a block + // the editor can open. + com.google.gson.JsonObject file = + com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks.findFile( + note.getValueJson(), blockId); + return file == null + ? null + : EditorAttachment.fromJsonObject(new JSONObject(file.toString())); } catch (Exception e) { Log.e(TAG, "findAttachmentByBlockId() failed", e); } diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 39ed2c8c..15891ab6 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -180,6 +180,16 @@ public void onInvalid(String errorMessage) { private SyncCoordinatorFactory.Result syncSetup; @Nullable private AccountSyncFragment accountTab; private boolean syncRunning; + + /** + * Set once a received settings version has scheduled this screen's recreation. + * + *

The controls stay usable for the moment the sync result is shown, and anything started in + * that moment — a sync, a sign-out, a conflict dialog — was torn down with the old instance, + * its callback dropped and the new instance showing "ready" while the sync still ran. + */ + private boolean recreatePending; + private SyncCoordinator syncCoordinator; private final ExecutorService syncExecutor = Executors.newSingleThreadExecutor(); @Inject FirebaseGoogleAuth firebaseGoogleAuth; @@ -297,7 +307,7 @@ private void renderSyncUi( } private void startSync() { - if (syncCoordinator == null) return; + if (syncCoordinator == null || recreatePending) return; if (!syncCoordinator.getProfile().isSignedIn()) { onInfoSnack( R.string.google_sign_in_failed, null, SnackBarInfo.Error, Snackbar.LENGTH_LONG); @@ -420,7 +430,7 @@ private static String formatBytes(long bytes) { } private void onGoogleSignInClicked() { - if (syncCoordinator == null) return; + if (syncCoordinator == null || recreatePending) return; if (syncCoordinator.getProfile().isSignedIn()) { syncCoordinator.disconnect( new SyncCoordinator.Callback() { @@ -501,16 +511,37 @@ private void applyReceivedPreferences() { null, SnackBarInfo.Success, Snackbar.LENGTH_LONG); + // Nothing may start while the rebuild is pending; see recreatePending. + recreatePending = true; + if (accountTab != null) accountTab.setSyncing(true); binding.getRoot() .postDelayed( () -> { - if (!isFinishing() && !isDestroyed()) { + if (!isFinishing() && !isDestroyed() && !syncRunning) { recreate(); } }, 1500L); } + /** + * Applies a restored theme once the restore has finished. + * + *

Applying it while the inserts were running recreated this screen and disposed them. + * Delayed so the result stays readable for a moment before the screen rebuilds, and only a mode + * that actually changed causes a rebuild at all. + */ + private void applyRestoredTheme() { + binding.getRoot() + .postDelayed( + () -> { + if (!isFinishing() && !isDestroyed()) { + themePreferencesCache.applyCurrentThemeMode(); + } + }, + 1500L); + } + private void finishSyncError(Exception error) { if (isFinishing() || isDestroyed()) { return; @@ -554,6 +585,7 @@ private void onBackgroundSyncToggled(boolean enabled) { } private void showNextConflictDialog() { + if (recreatePending) return; runInBackground( () -> { List unresolved = @@ -1035,12 +1067,11 @@ public void restoreFinish(int infoCode) { progressDialog.dismiss(); } switch (infoCode) { - case CloudErrors.OKAY_RESTORE -> - onInfoSnack( - R.string.restoreDataOkay, - null, - SnackBarInfo.Success, - Snackbar.LENGTH_LONG); + case CloudErrors.OKAY_RESTORE -> { + onInfoSnack( + R.string.restoreDataOkay, null, SnackBarInfo.Success, Snackbar.LENGTH_LONG); + applyRestoredTheme(); + } case CloudErrors.BACKUP_DESTROY -> onInfoSnack( R.string.restoreDataFall, diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java index 3bc05517..973b428f 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java @@ -7,7 +7,11 @@ import android.content.Context; import android.net.Uri; import android.util.Log; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.google.gson.Gson; +import com.pasich.mynotes.extendedEditor.attach.AttachmentStorage; +import com.pasich.mynotes.extendedEditor.attach.AttachmentUrl; import com.pasich.mynotes.utils.backup.models.JsonBackup; import java.io.ByteArrayOutputStream; import java.io.File; @@ -16,6 +20,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -25,6 +30,10 @@ public class ZipBackupHelper { private static final String TAG = "ZipBackupHelper"; + /** The only entry shape an archive may place a file under: one note folder, one file name. */ + private static final Pattern ATTACHMENT_ENTRY = + Pattern.compile(Pattern.quote(ATTACHMENTS_BASE_DIR) + "/(note_[1-9][0-9]*)/([^/]+)"); + /** Detect ZIP by magic header "PK" */ public static boolean isZip(byte[] data) { return data.length > 2 && data[0] == 0x50 && data[1] == 0x4B; @@ -76,79 +85,113 @@ public static File writeZipBackup(Context ctx, JsonBackup backup) throws Excepti return zipFile; } - /** Parse ZIP backup */ + /** + * Parses a ZIP backup, unpacking its attachments into the restore staging directory. + * + *

Nothing here touches a note's live attachment folder. The archive is untrusted input and + * the restore has not yet decided which row id each note will get, so files are staged and + * adopted per note once it is inserted; see {@code NoteAttachmentRelocator.adoptStaged}. + */ public static JsonBackup readZipBackup(Context ctx, Uri uri) throws Exception { - - JsonBackup backup = null; - + File staging = AttachmentStorage.restoreStagingDir(ctx); + // Whatever an earlier restore left behind belongs to that restore, not this one. + deleteRecursively(staging); try (InputStream is = ctx.getContentResolver().openInputStream(uri); ZipInputStream zis = new ZipInputStream(is)) { + JsonBackup backup = readZipBackup(zis, staging); + if (backup.isError()) { + deleteRecursively(staging); + } + return backup; + } + } - ZipEntry entry; - - while ((entry = zis.getNextEntry()) != null) { - - // ================== backup.json ================== - if (entry.getName().equals(FILE_NAME_BACKUP)) { - - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - byte[] tmp = new byte[4096]; - int n; - - while ((n = zis.read(tmp)) != -1) { - buffer.write(tmp, 0, n); - } - - String json = buffer.toString("UTF-8"); - backup = new Gson().fromJson(json, JsonBackup.class); + /** + * Filesystem-only core: reads the archive into {@code stagingRoot}, which stands in for the + * {@code attachments} directory an archive entry names. + */ + @NonNull + static JsonBackup readZipBackup(@NonNull ZipInputStream zis, @NonNull File stagingRoot) + throws IOException { + JsonBackup backup = null; + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.getName().equals(FILE_NAME_BACKUP)) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] tmp = new byte[4096]; + int n; + while ((n = zis.read(tmp)) != -1) { + buffer.write(tmp, 0, n); } - - // ================== attachments/... ================== - else if (entry.getName().startsWith(ATTACHMENTS_BASE_DIR)) { - - // A backup file is untrusted input: it can be edited, or come from - // somewhere else entirely. "attachments/../../databases/notes" also starts - // with the prefix above, so without resolving the path first an archive - // could write anywhere the app can write. - File out = safeAttachmentTarget(ctx, entry.getName()); - if (out == null || entry.isDirectory()) { - Log.w(TAG, "Skipping a backup entry outside the attachment directory"); - zis.closeEntry(); - continue; - } - File parent = out.getParentFile(); - if (parent != null && !parent.exists() && !parent.mkdirs()) { - Log.w(TAG, "Could not create the attachment directory for a backup entry"); - zis.closeEntry(); - continue; - } - - try (FileOutputStream fos = new FileOutputStream(out)) { - byte[] data = new byte[4096]; - int n; - - while ((n = zis.read(data)) != -1) { - fos.write(data, 0, n); - } + try { + backup = new Gson().fromJson(buffer.toString("UTF-8"), JsonBackup.class); + } catch (RuntimeException malformed) { + Log.w(TAG, "The backup JSON could not be read", malformed); + backup = null; + } + } else if (entry.getName().startsWith(ATTACHMENTS_BASE_DIR)) { + // A backup file is untrusted input: it can be edited, or come from somewhere else + // entirely. Only "attachments/note_/" is a place a restored note can + // reference; anything else — a traversal, a nested path, a folder the cleaner + // would never look in — is skipped rather than written somewhere and forgotten. + File out = stagedAttachmentTarget(stagingRoot, entry.getName()); + if (out == null || entry.isDirectory()) { + Log.w(TAG, "Skipping a backup entry outside the attachment directory"); + zis.closeEntry(); + continue; + } + File parent = out.getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + Log.w(TAG, "Could not create the attachment directory for a backup entry"); + zis.closeEntry(); + continue; + } + try (FileOutputStream fos = new FileOutputStream(out)) { + byte[] data = new byte[4096]; + int n; + while ((n = zis.read(data)) != -1) { + fos.write(data, 0, n); } } - - zis.closeEntry(); } + zis.closeEntry(); } - return backup != null ? backup : new JsonBackup().error(); } /** - * Resolves one archive entry inside the attachment directory, or {@code null} if it escapes. + * Resolves one archive entry inside the staging directory, or {@code null} if it does not name + * exactly one file in one note folder. * * @param entryName the raw name from the archive, which is attacker-controlled. */ - private static File safeAttachmentTarget(Context ctx, String entryName) throws IOException { - File root = new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR).getCanonicalFile(); - File resolved = new File(ctx.getFilesDir(), entryName).getCanonicalFile(); + @Nullable + static File stagedAttachmentTarget(@NonNull File stagingRoot, @NonNull String entryName) + throws IOException { + java.util.regex.Matcher matcher = ATTACHMENT_ENTRY.matcher(entryName); + if (!matcher.matches() || !AttachmentUrl.isSafeSegment(matcher.group(2))) { + return null; + } + File root = stagingRoot.getCanonicalFile(); + File resolved = + new File(new File(root, matcher.group(1)), matcher.group(2)).getCanonicalFile(); + // The pattern already forbids traversal; this is the second of two independent guards. String prefix = root.getPath() + File.separator; return resolved.getPath().startsWith(prefix) ? resolved : null; } + + private static void deleteRecursively(@Nullable File file) { + if (file == null || !file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + if (!file.delete()) { + Log.w(TAG, "Could not remove " + file.getName()); + } + } } diff --git a/app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java b/app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java new file mode 100644 index 00000000..baeae2e2 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java @@ -0,0 +1,55 @@ +package com.pasich.mynotes.data.preferences; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.pasich.mynotes.cache.AppPreferencesCache; +import com.pasich.mynotes.cache.ThemePreferencesCache; +import com.pasich.mynotes.utils.backup.models.PreferencesBackup; +import org.junit.Before; +import org.junit.Test; + +/** + * When a stored theme mode is pushed to AppCompat. + * + *

Pushing it recreates every started activity. That is wanted for a theme arriving with a sync + * and fatal during a backup restore, whose inserts are still running on the Backup screen: the + * recreation disposed them and left the database half restored with no message. + */ +public class AppPreferencesHelperTest { + + private ThemePreferencesCache themeCache; + private AppPreferencesHelper helper; + + @Before + public void setUp() { + AppPreferencesCache appCache = mock(AppPreferencesCache.class); + themeCache = mock(ThemePreferencesCache.class); + SafePreferences prefs = mock(SafePreferences.class); + when(prefs.commitAll(any())).thenReturn(true); + helper = new AppPreferencesHelper(appCache, themeCache, prefs, Runnable::run); + } + + @Test + public void aRestoreStoresTheThemeWithoutApplyingItMidWay() { + helper.setListPreferences(backup()); + + verify(themeCache).refresh(); + verify(themeCache, never()).applyCurrentThemeMode(); + } + + @Test + public void aSyncAppliesAReceivedThemeAtOnce() { + helper.commitListPreferences(backup()); + + verify(themeCache).applyCurrentThemeMode(); + } + + private static PreferencesBackup backup() { + return new PreferencesBackup( + 1, "sans", "date", 14, 11, false, 2, false, false, false, 1.0f); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java new file mode 100644 index 00000000..ebe1a4d6 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java @@ -0,0 +1,104 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Every snapshot build used to re-hash every attachment in the library; the cache is what makes an + * idle sync cheap. Correctness must never depend on it, so the tests also pin when it forgets. + */ +public class AttachmentHashCacheTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private final AtomicInteger hashed = new AtomicInteger(); + private final AttachmentHashCache.Hasher counting = + file -> { + hashed.incrementAndGet(); + return Sha256.of(file); + }; + + @Test + public void hashesAFileOnceWhileItIsUnchanged() throws Exception { + File file = write("photo.png", "bytes"); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + + String first = cache.sha256(file, counting); + String second = cache.sha256(file, counting); + + assertThat(second).isEqualTo(first); + assertThat(first).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(1); + } + + @Test + public void rehashesAFileWhoseContentChanged() throws Exception { + File file = write("photo.png", "bytes"); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + cache.sha256(file, counting); + + Files.write(file.toPath(), "different length".getBytes(StandardCharsets.UTF_8)); + String rehashed = cache.sha256(file, counting); + + assertThat(rehashed).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(2); + } + + @Test + public void survivesTheStoreInstanceThatBuiltIt() throws Exception { + // A store lives for one sync; without persistence the first build of every sync — and + // the estimate before the first one — hashed the whole library again. + File file = write("photo.png", "bytes"); + AttachmentHashCache first = new AttachmentHashCache(storage()); + first.sha256(file, counting); + first.flush(); + + AttachmentHashCache second = new AttachmentHashCache(storage()); + String hash = second.sha256(file, counting); + + assertThat(hash).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(1); + } + + @Test + public void anUnreadableCacheFileCostsARehashNotAFailure() throws Exception { + File file = write("photo.png", "bytes"); + assertThat(storage().getParentFile().mkdirs()).isTrue(); + Files.write(storage().toPath(), "{not json".getBytes(StandardCharsets.UTF_8)); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + + assertThat(cache.sha256(file, counting)).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(1); + } + + @Test + public void clearForgetsEverything() throws Exception { + File file = write("photo.png", "bytes"); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + cache.sha256(file, counting); + cache.flush(); + + cache.clear(); + + assertThat(storage().exists()).isFalse(); + cache.sha256(file, counting); + assertThat(hashed.get()).isEqualTo(2); + } + + private File storage() { + return new File(temporaryFolder.getRoot(), "sync-attachments/hash-cache.json"); + } + + private File write(String name, String content) throws Exception { + File file = new File(temporaryFolder.getRoot(), name); + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return file; + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java new file mode 100644 index 00000000..7184b559 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java @@ -0,0 +1,31 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +public class AttachmentWireUrlTest { + + private static final String ID = "7d444840-9dc0-11d1-b245-5ffdce74fad2"; + + @Test + public void roundTripsALogicalId() { + String wire = AttachmentWireUrl.forLogicalId(ID); + + assertThat(AttachmentWireUrl.logicalIdOf(wire)).isEqualTo(ID); + } + + @Test + public void isNotMistakenForALocalReference() { + // A wire reference must never parse as a note-folder path, or a leaked one would be + // taken for a file this device owns. + assertThat( + com.pasich.mynotes.extendedEditor.attach.AttachmentUrl.parse( + AttachmentWireUrl.forLogicalId(ID))) + .isNull(); + assertThat(AttachmentWireUrl.logicalIdOf("editorjs://attachments/note_5/photo.png")) + .isNull(); + assertThat(AttachmentWireUrl.logicalIdOf("mynotes-sync://attachment/not-a-uuid")).isNull(); + assertThat(AttachmentWireUrl.logicalIdOf(null)).isNull(); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 2d4b132f..5ae9bbd4 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -538,10 +538,16 @@ public void publishingWithoutAPrecedingReadIsRefused() throws Exception { GoogleDriveSyncBackend backend = backend(); try { - backend.writeSnapshot(snapshot(NOTE_ID, null)); + // A context that no read of this backend produced: the token is empty. + backend.publish( + new SyncPublication( + snapshot(NOTE_ID, null), + Collections.emptyList(), + Collections.emptySet(), + RemoteSnapshot.of(snapshot(NOTE_ID, null)))); throw new AssertionError("Expected a publish with no read context to be refused"); } catch (IOException expected) { - assertThat(expected).hasMessageThat().contains("read context"); + assertThat(expected).hasMessageThat().contains("latest remote read"); } } @@ -589,6 +595,272 @@ public void aDeletedAncestorBundleDoesNotBreakSyncOrLoseAnAlternative() throws E .isEqualTo("written on A"); } + // ------------------------------------------------- transport failure classification + + @Test + public void mayHaveCommitted_treatsAReadTimeoutAsAmbiguous() { + // SocketTimeoutException extends InterruptedIOException, and the old classifier asked + // about the parent first, so a timeout waiting for the response of an upload that had + // already landed failed the sync instead of being confirmed by discovery. + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(new java.net.SocketTimeoutException())) + .isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(new java.io.InterruptedIOException())) + .isFalse(); + } + + @Test + public void mayHaveCommitted_agreesWithTheRetryPolicyAboutTransientStatuses() { + // One answer for bundle and attachment uploads: a 429 used to be rediscovered for the + // bundle POST and rethrown for the attachment POST. + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(429, ""))).isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(503, ""))).isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(403, "rateLimitExceeded"))) + .isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(403, "forbidden"))).isFalse(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(401, ""))).isFalse(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(400, ""))).isFalse(); + assertThat( + GoogleDriveSyncBackend.mayHaveCommitted( + new AttachmentIntegrityException("checksum"))) + .isFalse(); + } + + private static IOException http(int status, String detail) { + return new DriveRequestExecutor.DriveHttpException(status, null, detail); + } + + // ------------------------------------------------- bundle history + + @Test + public void validateAncestry_walksALongLinearHistoryWithoutOverflowingTheStack() + throws Exception { + // An account that syncs after every edit builds exactly this shape. The recursive walk + // used one frame per ancestor and a StackOverflowError is not an IOException the sync + // knows how to report. + Map> parents = new LinkedHashMap<>(); + String previous = null; + for (int index = 0; index < 200_000; index++) { + String id = "bundle-" + index; + parents.put(id, previous == null ? Collections.emptyList() : List.of(previous)); + previous = id; + } + Throwable[] failure = new Throwable[1]; + Thread small = + new Thread( + null, + () -> { + try { + GoogleDriveSyncBackend.validateAncestry(parents); + } catch (Throwable error) { + failure[0] = error; + } + }, + "small-stack", + 256L * 1024L); + small.start(); + small.join(30_000L); + + assertThat(small.isAlive()).isFalse(); + assertThat(failure[0]).isNull(); + } + + @Test + public void validateAncestry_stillRejectsACycle() { + Map> parents = new LinkedHashMap<>(); + parents.put("a", List.of("b")); + parents.put("b", List.of("c")); + parents.put("c", List.of("a", "missing")); + + try { + GoogleDriveSyncBackend.validateAncestry(parents); + throw new AssertionError("Expected the cycle to be refused"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("cycle"); + } + } + + @Test + public void publish_namesTheFrontierOfTheReadItQuotesAsParents() throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + server.seedOwnedBundleBytes(base); + GoogleDriveSyncBackend backend = backend(); + RemoteSnapshot context = backend.readSnapshotResult(); + + backend.publish( + new SyncPublication( + snapshotWithTitle("Next"), + Collections.emptyList(), + Collections.emptySet(), + context)); + + // The parents come from the quoted read, not from a second copy of its frontier kept on + // the backend that had to be kept in step by hand. + assertThat(context.getFrontierBundleIds()).containsExactly(baseId); + assertThat( + codec.decode(new ByteArrayInputStream(server.newestBundleBytes())) + .getParentBundleIds()) + .containsExactly(baseId); + } + + @Test + public void publish_retiresTheBundlesTheNewOneSupersedes() throws Exception { + // Nothing ever deleted a bundle, so every sync downloaded and decoded the whole history + // to find one or two heads. Seeded bundles carry no publication time, which counts as old. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] first = + codec.encode( + snapshotWithTitle("First"), CLOCK.instant(), Collections.singleton(baseId)); + String firstId = codec.decode(new ByteArrayInputStream(first)).getBundleId(); + byte[] second = + codec.encode( + snapshotWithTitle("Second"), + CLOCK.instant(), + Collections.singleton(firstId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(first); + server.seedOwnedBundleBytes(second); + + publish(backend(), snapshotWithTitle("Third")); + + // The head the publish descended from is kept: a concurrent publisher is about to name + // it as a parent. Its ancestors are gone. + assertThat(server.bundleCount()).isEqualTo(2); + assertThat(server.deletedFileIds()).hasSize(2); + assertThat( + backend() + .readSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID) + .getPayload() + .get("title") + .getAsString()) + .isEqualTo("Third"); + } + + @Test + public void publish_keepsASupersededBundleUntilTheGracePeriodHasPassed() throws Exception { + GoogleDriveSyncBackend recent = backend(); + publish(recent, snapshotWithTitle("First")); + publish(recent, snapshotWithTitle("Second")); + + // Both were published by this clock moments ago; a device that listed the folder just + // before may still be reading the older one. + assertThat(server.bundleCount()).isEqualTo(2); + + GoogleDriveSyncBackend later = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + Clock.offset(CLOCK, java.time.Duration.ofHours(2)), + new SyncBundleCodec()); + publish(later, snapshotWithTitle("Third")); + + // Two hours on, the first is an ancestor nobody can still be fetching; the second is the + // head this publish descended from and stays for one more round. + assertThat(server.bundleCount()).isEqualTo(2); + assertThat(server.deletedFileIds()).hasSize(1); + } + + // ------------------------------------------------- attachment transfer cost + + @Test + public void readAttachment_downloadsTheBlobOnce() throws Exception { + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + + try (java.io.InputStream stream = backend().readAttachment(hash)) { + assertThat(readAll(stream)).isEqualTo(bytes); + } + + // It used to be downloaded in full to pick a verified candidate and then downloaded + // again to hand over; every caller verifies the stream it receives anyway. + assertThat(server.mediaReadsOfAttachment(hash)).isEqualTo(1); + } + + @Test + public void hasVerifiedAttachment_trustsDrivesOwnChecksumWithoutDownloading() throws Exception { + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + byte[] wrong = "wrong bytes".getBytes(StandardCharsets.UTF_8); + String claimed = sha256("something else".getBytes(StandardCharsets.UTF_8)); + server.seedCorruptAttachment(claimed, wrong); + + GoogleDriveSyncBackend backend = backend(); + + // Every attachment in the account used to be re-downloaded on every sync just to answer + // this; Drive computes the digest over the stored bytes, so a mismatch shows in the + // listing as well. + assertThat(backend.hasVerifiedAttachment(hash, (long) bytes.length)).isTrue(); + assertThat(backend.hasVerifiedAttachment(claimed, (long) wrong.length)).isFalse(); + assertThat(server.mediaReadsOfAttachment(hash)).isEqualTo(0); + assertThat(server.mediaReadsOfAttachment(claimed)).isEqualTo(0); + } + + @Test + public void oneSyncListsTheRootFoldersOnce() throws Exception { + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + GoogleDriveSyncBackend backend = backend(); + int before = server.folderListings(); + + backend.readSnapshotResult(); + backend.hasAttachment(hash); + backend.hasVerifiedAttachment(hash, (long) bytes.length); + try (java.io.InputStream stream = backend.readAttachment(hash)) { + readAll(stream); + } + + // Each of those used to list the roots again; with N attachments that was several times + // N listings per sync before a byte moved, which is what Drive rate-limited. + assertThat(server.folderListings() - before).isEqualTo(1); + } + + @Test + public void anOversizedCandidateIsSkippedRatherThanFailingTheSync() throws Exception { + // A Drive object larger than the ceiling and tagged with the expected hash: the ceiling + // used to throw a plain IOException from the stream, past the "skip a corrupt candidate" + // path, so one bad object failed every sync. Drive is asked to withhold its checksum so + // the bytes have to be read, which is what the ceiling guards. + byte[] good = "good".getBytes(StandardCharsets.UTF_8); + String hash = sha256(good); + server.seedCorruptAttachment(hash, new byte[64]); + server.withholdChecksums(); + GoogleDriveSyncBackend backend = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + CLOCK, + new SyncBundleCodec(), + 16L); + + assertThat(backend.hasVerifiedAttachment(hash, (long) good.length)).isFalse(); + backend.writeAttachment(hash, good.length, new ByteArrayInputStream(good)); + + assertThat(server.ownedAttachmentCount(hash)).isEqualTo(2); + try (java.io.InputStream restored = backend.readAttachment(hash)) { + assertThat(readAll(restored)).isEqualTo(good); + } + } + + @Test + public void writeAttachment_withAnUnknownSizeBuffersWithinTheCeilingAndUploads() + throws Exception { + byte[] bytes = "size unknown".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + + backend().writeAttachment(hash, -1L, new ByteArrayInputStream(bytes)); + + assertThat(server.attachmentContent(hash)).isEqualTo(bytes); + } + private static final java.time.Instant T10 = java.time.Instant.parse("2026-08-31T12:00:10Z"); private static final java.time.Instant T20 = java.time.Instant.parse("2026-08-31T12:00:20Z"); @@ -888,6 +1160,11 @@ private static final class FakeDriveServer implements AutoCloseable { java.util.Collections.synchronizedList(new ArrayList<>()); private volatile boolean running = true; private volatile CyclicBarrier emptyRootListingBarrier; + private final AtomicInteger folderListings = new AtomicInteger(); + private final Map mediaReads = new ConcurrentHashMap<>(); + private final List deletedFileIds = + java.util.Collections.synchronizedList(new ArrayList<>()); + private volatile boolean withholdChecksums; private SyncSnapshot updateBeforeNextPatch; private final AtomicInteger nextId = new AtomicInteger(1); @@ -953,6 +1230,36 @@ List rejectedChunkRanges() { return new ArrayList<>(rejectedChunkRanges); } + /** How many times the folder index was listed. */ + int folderListings() { + return folderListings.get(); + } + + /** How many times the bytes of the blob carrying {@code sha256} were downloaded. */ + int mediaReadsOfAttachment(String sha256) { + int total = 0; + for (DriveFile file : files.values()) { + if (sha256.equals(file.appProperties.get("mynotesAttachmentSha256"))) { + total += mediaReads.getOrDefault(file.id, 0); + } + } + return total; + } + + /** Ids of files the client deleted. */ + List deletedFileIds() { + return new ArrayList<>(deletedFileIds); + } + + /** Marks every stored bundle as published {@code millis} ago relative to {@code now}. */ + void ageBundles(long now, long millis) { + for (DriveFile file : files.values()) { + if ("1".equals(file.appProperties.get("mynotesBundle"))) { + file.appProperties.put("mynotesBundlePublishedAt", Long.toString(now - millis)); + } + } + } + /** Committed bytes of the attachment blob carrying {@code sha256}, or null. */ byte[] attachmentContent(String sha256) { for (DriveFile file : files.values()) { @@ -1047,6 +1354,24 @@ byte[] readBundleBytes() { return null; } + /** The most recently created bundle file's bytes. */ + byte[] newestBundleBytes() { + DriveFile newest = null; + for (DriveFile file : files.values()) { + if ("1".equals(file.appProperties.get("mynotesBundle")) + && (newest == null + || Integer.parseInt(file.id) > Integer.parseInt(newest.id))) { + newest = file; + } + } + return newest == null ? null : newest.content; + } + + /** Models objects Drive has not (yet) checksummed: listings carry no digest or size. */ + void withholdChecksums() { + withholdChecksums = true; + } + String registerAttachment(byte[] bytes) throws Exception { String hash = sha256(bytes); seededAttachmentContent.put(hash, bytes); @@ -1144,7 +1469,7 @@ private Response dispatch(Request request) throws IOException { return Response.json(405, "{}"); } if (path.startsWith("/drive/v3/files/")) { - return handleFileRead(uri, path.substring("/drive/v3/files/".length())); + return handleFileRead(request, uri, path.substring("/drive/v3/files/".length())); } if ("/upload/drive/v3/files".equals(path) && "POST".equals(request.method)) { if ("resumable".equals(parseQuery(uri).get("uploadType"))) { @@ -1175,12 +1500,30 @@ && ownedFolderCount() == 0) { return Response.json(500, "{}"); } } + if (query != null + && query.contains("mimeType = 'application/vnd.google-apps.folder'")) { + folderListings.incrementAndGet(); + } JsonArray array = new JsonArray(); for (DriveFile file : files.values()) { if (matchesQuery(file, query)) { JsonObject value = new JsonObject(); value.addProperty("id", file.id); value.addProperty("name", file.name); + // Drive reports these for binary files; the fake computes them from the + // stored bytes exactly as Drive does, so a corrupt object is exposed by its + // real digest rather than by the property the uploader claimed. + if (!withholdChecksums) { + value.addProperty("size", String.valueOf(file.content.length)); + if (!"application/vnd.google-apps.folder".equals(file.mimeType)) { + value.addProperty("sha256Checksum", sha256Unchecked(file.content)); + } + } + JsonObject appProperties = new JsonObject(); + for (Map.Entry entry : file.appProperties.entrySet()) { + appProperties.addProperty(entry.getKey(), entry.getValue()); + } + value.add("appProperties", appProperties); array.add(value); } } @@ -1189,6 +1532,14 @@ && ownedFolderCount() == 0) { return Response.json(200, response.toString()); } + private static String sha256Unchecked(byte[] bytes) { + try { + return sha256(bytes); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + private Response handleCreateMetadata(byte[] body) throws IOException { JsonObject metadata = readJson(body); DriveFile file = @@ -1200,12 +1551,18 @@ private Response handleCreateMetadata(byte[] body) throws IOException { return Response.json(200, fileMetadata(file).toString(), file.eTag()); } - private Response handleFileRead(URI uri, String id) { + private Response handleFileRead(Request request, URI uri, String id) { DriveFile file = files.get(id); if (file == null) { return Response.json(404, "{}"); } + if ("DELETE".equals(request.method)) { + files.remove(id); + deletedFileIds.add(id); + return Response.json(204, ""); + } if ("media".equals(parseQuery(uri).get("alt"))) { + mediaReads.merge(id, 1, Integer::sum); return Response.binary(200, file.content, file.eTag()); } return Response.json(200, fileMetadata(file).toString(), file.eTag()); @@ -1293,6 +1650,13 @@ private Response handleResumableChunk(Request request, String sessionId) rejectedChunkRanges.add(range); return Response.json(400, "{}"); } + if (end + 1L < session.totalBytes && request.body.length % (256 * 1024) != 0) { + // Drive's rule: every chunk but the last is a multiple of 256 KiB. A partially + // acknowledged window used to be continued with just its tail, which Drive + // answers with a 400 nothing retries. + rejectedChunkRanges.add(range); + return Response.json(400, "{}"); + } if (script != null && script.forcedRangeInclusiveEnd != null) { Map headers = new LinkedHashMap<>(); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java new file mode 100644 index 00000000..78a38ceb --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java @@ -0,0 +1,51 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +/** + * Whether the live settings count as a local edit. + * + *

The baseline changed format once — 2.6.49 stored the payload JSON, later releases its digest — + * and every device upgrading across that boundary claimed a local edit it never made, letting its + * untouched settings outrank a genuine change from another device. + */ +public class PreferencesBaselineDecisionTest { + + private static final String LEGACY_JSON = "{\"a\":1,\"b\":14,\"c\":11,\"g\":true}"; + private static final String DIGEST = Sha256.of(LEGACY_JSON); + + @Test + public void unchangedSettingsAreNotAnEdit() { + assertThat(PreferencesBaselineDecision.decide(DIGEST, DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.UNCHANGED); + } + + @Test + public void aLegacyBaselineDescribingTheLiveValuesIsMigratedNotTreatedAsAnEdit() { + assertThat(PreferencesBaselineDecision.decide(LEGACY_JSON, DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.MIGRATE_BASELINE); + } + + @Test + public void aLegacyBaselineForOtherValuesIsAnEdit() { + assertThat( + PreferencesBaselineDecision.decide( + "{\"a\":1,\"b\":14,\"c\":2,\"g\":true}", DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.LOCAL_EDIT); + } + + @Test + public void aDifferentDigestIsAnEdit() { + assertThat(PreferencesBaselineDecision.decide(Sha256.of("other"), DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.LOCAL_EDIT); + } + + @Test + public void noBaselineAtAllIsAnEdit() { + // Sync has never seen these settings; publishing them is the conservative reading. + assertThat(PreferencesBaselineDecision.decide(null, DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.LOCAL_EDIT); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java b/app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java new file mode 100644 index 00000000..177c60e2 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java @@ -0,0 +1,59 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Locale; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * The one digest every sync component hashes with. + * + *

A blob hashed by the store has to match a manifest written by the validator and a checksum + * compared by the backend, so the encoding is pinned rather than left to whichever of the former + * five copies a caller happened to reach. + */ +public class Sha256Test { + + private static final String EMPTY = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void everyOverloadAgreesOnTheKnownVector() throws Exception { + File file = temporaryFolder.newFile("empty"); + Files.write(file.toPath(), new byte[0]); + + assertThat(Sha256.of(new byte[0])).isEqualTo(EMPTY); + assertThat(Sha256.of("")).isEqualTo(EMPTY); + assertThat(Sha256.of(new ByteArrayInputStream(new byte[0]))).isEqualTo(EMPTY); + assertThat(Sha256.of(file)).isEqualTo(EMPTY); + assertThat(Sha256.of("abc".getBytes(StandardCharsets.UTF_8))) + .isEqualTo("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + @Test + public void hexIsLowercaseAndZeroPadded() { + assertThat(Sha256.hex(new byte[] {0, (byte) 0xff, 0x10, 0x0a})).isEqualTo("00ff100a"); + } + + @Test + public void hexDoesNotDependOnTheDefaultLocale() { + // Three of the former copies formatted through String.format("%02x") with the default + // locale; a locale with its own digits would have made a locally hashed blob mismatch the + // manifest it was validated against. + Locale previous = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("ar-EG")); + assertThat(Sha256.of("")).isEqualTo(EMPTY); + } finally { + Locale.setDefault(previous); + } + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java new file mode 100644 index 00000000..4bf9b45b --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java @@ -0,0 +1,59 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.Collections; +import org.junit.Test; + +public class SnapshotProblemTest { + + private static final String NOTE_ID = "550e8400-e29b-41d4-a716-446655440000"; + + @Test + public void theFailureNamesTheNoteItIsIn() throws java.io.IOException { + // "MISSING_ATTACHMENT" alone left the user to guess which of their notes to open; the + // account screen shows this string as the sync status. + SnapshotBuildResult result = + SnapshotBuildResult.incomplete( + SyncSnapshot.empty(), + Collections.singletonList( + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + "Shopping list"))); + + try { + result.requireSnapshot(); + throw new AssertionError("Expected an incomplete snapshot to be refused"); + } catch (SnapshotBuildResult.SnapshotBuildException expected) { + assertThat(expected) + .hasMessageThat() + .isEqualTo( + "Local snapshot is incomplete: MISSING_ATTACHMENT in note" + + " \"Shopping list\""); + } + } + + @Test + public void aLongTitleIsCutShortAndABlankOneIsDropped() { + String title = "x".repeat(80); + + SnapshotProblem long_ = + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + title); + SnapshotProblem blank = + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + " "); + + assertThat(long_.getLabel()).hasLength(41); + assertThat(blank.getLabel()).isNull(); + assertThat(blank.describe()).isEqualTo("MISSING_ATTACHMENT"); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index e157bcbb..c58c6bf5 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -303,6 +303,112 @@ public void encode_dropsEmptyAttachmentFieldsInsteadOfLeakingThemToTheWire() thr .doesNotContain("attachmentNames"); } + @Test + public void encode_carriesAnAlternativeWhoseAttachmentIdMapsToDifferentContent() + throws Exception { + // A live note and one of its unresolved alternatives can describe different bytes under + // one logical attachment id — the file was replaced in place, or two devices derived the + // same id. Refusing to encode that failed every publish for the account, before the + // conflict could even be stored for the user to settle. + String otherHash = "0000000000000000000000000000000000000000000000000000000000000001"; + // Both payloads in the shape RoomSyncStore builds, names included. + JsonObject livePayload = notePayload("Live", "image/png", 42L, "receipt.png"); + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "receipt.png"); + livePayload.add("attachmentNames", names.deepCopy()); + SyncRecord live = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:05Z"), + livePayload); + JsonObject alternativePayload = notePayload("Older", "image/png", 77L, "receipt.png"); + alternativePayload.add("attachmentNames", names.deepCopy()); + alternativePayload + .getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("sha256", otherHash); + alternativePayload + .getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("path", "attachments/" + otherHash); + alternativePayload + .getAsJsonArray("attachmentHashes") + .set(0, new com.google.gson.JsonPrimitive(otherHash)); + SyncRecord alternative = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:04Z"), + alternativePayload); + + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(java.util.Collections.singletonList(live)), + CREATED_AT, + java.util.Collections.emptyList(), + java.util.Collections.singletonList(alternative), + java.util.Collections.emptySet()); + SyncBundleCodec.DecodedBundle decoded = codec.decode(new ByteArrayInputStream(bundle)); + + // Both blobs are described, and the decoded alternative is the version that went in: + // same logical id, same content, same identity for the resolution bookkeeping. + assertThat(decoded.getAttachmentsByHash().keySet()).containsExactly(HASH, otherHash); + SyncRecord decodedAlternative = decoded.getAlternatives().get(0); + JsonObject entry = + decodedAlternative + .getPayload() + .getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject(); + assertThat(entry.get("id").getAsString()).isEqualTo(ATTACHMENT_ID); + assertThat(entry.get("sha256").getAsString()).isEqualTo(otherHash); + assertThat(decodedAlternative.getCanonicalPayloadHash()) + .isEqualTo(alternative.getCanonicalPayloadHash()); + assertThat( + decoded.getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID) + .getCanonicalPayloadHash()) + .isEqualTo(live.getCanonicalPayloadHash()); + } + + @Test + public void decode_trimsADisplayNameSoEveryConsumerSeesTheNameThatWasValidated() + throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + SyncRecord local = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + notePayload("Body", "application/pdf", 12L, "report.pdf ")); + + SyncRecord decoded = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + java.util.Collections.singletonList(local)), + CREATED_AT))) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + + // The validator judged the trimmed name; a consumer building a file name from the + // untrimmed one would have been refused by the URL parser. + JsonObject entry = + decoded.getPayload().getAsJsonArray("attachmentsManifest").get(0).getAsJsonObject(); + assertThat(entry.get("displayName").getAsString()).isEqualTo("report.pdf"); + assertThat( + decoded.getPayload() + .getAsJsonObject("attachmentNames") + .get(ATTACHMENT_ID) + .getAsString()) + .isEqualTo("report.pdf"); + } + private static SyncRecord task(String title) { JsonObject payload = new JsonObject(); payload.addProperty("title", title); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java index f49f99ef..84298e21 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java @@ -249,13 +249,13 @@ public String getIdentifier() { } @Override - public SyncSnapshot readSnapshot() { - return snapshot; + public RemoteSnapshot readSnapshotResult() { + return RemoteSnapshot.of(snapshot); } @Override - public void writeSnapshot(SyncSnapshot snapshot) { - this.snapshot = snapshot; + public void publish(SyncPublication publication) { + this.snapshot = publication.getSnapshot(); } @Override diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java index 876fb0ec..10d15f81 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java @@ -74,8 +74,9 @@ public void insertNotes_insertsIdKeepingNotesBeforeReassignedOnes() { taken.setId(1); Note free = new Note().create("Beta", "body", 20L, ""); free.setId(2); - when(noteDao.getNoteSync(1)).thenReturn(new Note().create("Occupant", "", 5L, "")); - when(noteDao.getNoteSync(2)).thenReturn(null); + Note occupant = new Note().create("Occupant", "", 5L, ""); + occupant.setId(1); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(occupant)); when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())) .thenReturn(new long[] {2L}) .thenReturn(new long[] {3L}); @@ -119,7 +120,7 @@ public void insertNotes_skipsANoteThisDeviceAlreadyHasUnchanged() { existing.setId(5); Note fromBackup = new Note().create("Title", "Body", 10L, "work"); fromBackup.setId(5); - when(noteDao.getNoteSync(5)).thenReturn(existing); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(fromBackup))); @@ -134,7 +135,7 @@ public void insertNotes_keepsADifferentNoteThatHappensToShareARowId() { existing.setId(5); Note fromBackup = new Note().create("Other", "Other body", 20L, ""); fromBackup.setId(5); - when(noteDao.getNoteSync(5)).thenReturn(existing); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())).thenReturn(new long[] {77L}); coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(fromBackup))); @@ -303,7 +304,7 @@ public void insertNotes_keepsBackupIdsWhenNothingOccupiesThem() { Note restored = new Note().create("One", "1", 1L, ""); restored.setId(7); notes.add(restored); - when(noteDao.getNoteSync(7)).thenReturn(null); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); when(noteDao.addNotes(anyList())).thenReturn(new long[] {7L}); coordinator.insertNotes(notes); @@ -324,7 +325,7 @@ public void insertNotes_doesNotOverwriteAnExistingNoteThatHoldsTheSameId() { notes.add(restored); Note occupant = new Note().create("Already here", "x", 2L, ""); occupant.setId(7); - when(noteDao.getNoteSync(7)).thenReturn(occupant); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(occupant)); when(noteDao.addNotes(anyList())).thenReturn(new long[] {42L}); coordinator.insertNotes(notes); @@ -388,6 +389,108 @@ public void renameTag_touchesTagAndAllNotesChangedByRename() { .isEqualTo(1_000L); } + @Test + public void insertNotes_looksUpEveryIncomingIdInOneBatchInsteadOfOnePerNote() { + // Restoring a few thousand notes ran four point queries per note inside the transaction + // and sat on the restore dialog for tens of seconds. One IN (...) query per batch, and + // no lookup at all before the INSERT OR IGNORE of the metadata row. + List notes = new ArrayList<>(); + for (int id = 1; id <= 3; id++) { + Note note = new Note().create("Note " + id, "body", id, ""); + note.setId(id); + notes.add(note); + } + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {1L, 2L, 3L}); + + coordinator.insertNotes(notes); + + verify(noteDao, org.mockito.Mockito.times(1)).getNotesByIdsSync(anyList()); + verify(noteDao, never()).getNoteSync(org.mockito.ArgumentMatchers.anyInt()); + assertThat(syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 3L)).isNotNull(); + } + + @Test + public void insertNotes_keepsBothNotesWhenTheBackupItselfRepeatsAnId() { + // A backup made by concatenating two exports can carry one id twice. addNotes is a + // REPLACE insert, so the second note used to overwrite the first inside the same call + // — no error, no duplicate, one note gone. + Note first = new Note().create("First", "body", 10L, ""); + first.setId(12); + Note second = new Note().create("Second", "body", 20L, ""); + second.setId(12); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {12L}).thenReturn(new long[] {13L}); + + coordinator.insertNotes(new ArrayList<>(List.of(first, second))); + + verify(noteDao, org.mockito.Mockito.times(2)).addNotes(anyList()); + assertThat(first.getId()).isEqualTo(12); + assertThat(second.getId()).isEqualTo(13); + } + + @Test + public void insertTags_keepsBothTagsWhenTheBackupItselfRepeatsAnId() { + Tag first = new Tag().create("work"); + first.id = 4; + Tag second = new Tag().create("home"); + second.id = 4; + when(tagsDao.getTagsByIdsSync(anyList())).thenReturn(List.of()); + when(tagsDao.addTags(anyList())).thenReturn(new long[] {4L}).thenReturn(new long[] {5L}); + + coordinator.insertTags(new ArrayList<>(List.of(first, second))); + + assertThat(first.getId()).isEqualTo(4L); + assertThat(second.getId()).isEqualTo(5L); + } + + @Test + public void insertNotes_adoptsStagedAttachmentsEvenWhenTheIdIsKept() { + // The archive's files wait in a staging directory until the row exists; a note that + // keeps its id still has to have them moved into its folder, so the relocation runs for + // every restored note and the content is stored again only when it rewrote something. + List relocatedFrom = new ArrayList<>(); + SyncMutationCoordinator staging = + new SyncMutationCoordinator( + new SyncMutationCoordinator.TransactionExecutor() { + @Override + public T run( + SyncMutationCoordinator.TransactionCallable callable) { + return callable.call(); + } + }, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + new FixedTimeProvider(1_000L), + new QueueStableIdGenerator("stable-a"), + (note, previousId) -> { + relocatedFrom.add(previousId); + note.setAttachments("[moved]"); + return true; + }); + Note restored = new Note().create("Kept", "body", 1L, ""); + restored.setId(7); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {7L}); + + staging.insertNotes(new ArrayList<>(List.of(restored))); + + assertThat(relocatedFrom).containsExactly(7); + verify(noteDao) + .updateNoteContent( + eq(7), + eq("Kept"), + eq("body"), + org.mockito.ArgumentMatchers.any(), + eq(1L), + eq(""), + eq("[moved]")); + } + private static final class FixedTimeProvider implements SyncMutationCoordinator.TimeProvider { private final long value; @@ -446,6 +549,15 @@ public boolean exists(String recordType, long localId) { return rows.containsKey(key(recordType, localId)); } + @Override + public List getExistingLocalIds(String recordType, List localIds) { + List existing = new ArrayList<>(); + for (Long localId : localIds) { + if (exists(recordType, localId)) existing.add(localId); + } + return existing; + } + @Override public SyncMetadataEntity getByStableId(String recordType, String stableId) { for (SyncMetadataEntity value : rows.values()) { diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java new file mode 100644 index 00000000..99db5a2a --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java @@ -0,0 +1,45 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.time.Instant; +import org.junit.Test; + +public class SyncRecordTest { + + private static final String NOTE_ID = "550e8400-e29b-41d4-a716-446655440000"; + private static final Instant AT = Instant.parse("2026-08-31T12:00:01Z"); + + @Test + public void aNoteWithEmptyAttachmentFieldsHashesLikeOneWithout() { + // The local build, the encoder and the decoder each enforced "absent when empty" on + // their own, and every time one drifted the same note hashed differently on the two + // sides and conflicted with itself on every sync. The record is the one place now. + JsonObject bare = new JsonObject(); + bare.addProperty("b", "Shopping"); + JsonObject withEmpties = bare.deepCopy(); + withEmpties.add("attachmentsManifest", new JsonArray()); + withEmpties.add("attachmentHashes", new JsonArray()); + withEmpties.add("attachmentNames", new JsonObject()); + + SyncRecord plain = SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, AT, bare); + SyncRecord normalized = SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, AT, withEmpties); + + assertThat(normalized.getCanonicalPayloadHash()).isEqualTo(plain.getCanonicalPayloadHash()); + assertThat(normalized.getPayload().has("attachmentNames")).isFalse(); + } + + @Test + public void populatedAttachmentFieldsAreKept() { + JsonObject payload = new JsonObject(); + JsonArray hashes = new JsonArray(); + hashes.add("d6f1f3d5d8cf9b5a4a2469787998dc45eb59f401b93b1b4cde4998dc409ebdc8"); + payload.add("attachmentHashes", hashes); + + SyncRecord record = SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, AT, payload); + + assertThat(record.getPayload().getAsJsonArray("attachmentHashes")).hasSize(1); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index 7f7f40d5..5ece616e 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -305,6 +305,62 @@ public void sync_replacesCorruptLocalAttachmentWithVerifiedRemoteBlob() throws E assertThat(backend.writeSnapshotCalls).isEqualTo(1); } + @Test + public void runWhileNoSyncRuns_waitsForTheSyncInFlightToFinish() throws Exception { + // Disconnect wipes state, conflicts and the blob cache. Done concurrently with the + // six-hourly worker it deleted the cache under the worker, which then wrote the old + // account's state and conflicts back after the wipe. + FakeStore store = new FakeStore(snapshot(note(TEN, "Local"))); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + backend.readStarted = new java.util.concurrent.CountDownLatch(1); + backend.readReleased = new java.util.concurrent.CountDownLatch(1); + List order = Collections.synchronizedList(new ArrayList<>()); + Thread syncing = + new Thread( + () -> { + new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + order.add("sync finished"); + }); + syncing.start(); + assertThat(backend.readStarted.await(5, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + + Thread clearing = + new Thread(() -> SyncService.runWhileNoSyncRuns(() -> order.add("cleared"))); + clearing.start(); + Thread.sleep(200L); + + // The sync still holds the lock, so the wipe has not run. + assertThat(order).isEmpty(); + backend.readReleased.countDown(); + syncing.join(5_000L); + clearing.join(5_000L); + assertThat(order).containsExactly("sync finished", "cleared").inOrder(); + assertThat(store.appliedSnapshot).isNotNull(); + } + + @Test + public void sync_pinsConflictBlobsWithoutTransferringOrVerifyingThemAgain() throws Exception { + byte[] bytes = "shared attachment".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + SyncRecord local = noteWithAttachment(TEN, "Local wording", hash, bytes.length); + SyncRecord remote = noteWithAttachment(TWENTY, "Remote wording", hash, bytes.length); + FakeStore store = new FakeStore(snapshot(local)); + store.attachmentHashes = Collections.singletonList(hash); + store.attachments.put(hash, bytes); + FakeBackend backend = new FakeBackend(snapshot(remote)); + backend.attachments.put(hash, bytes); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(state.getConflictCount()).isEqualTo(1); + // The blob was verified once for the merged snapshot. Pinning the winner and the loser + // used to run the whole transfer again for each and copy the cached blob onto itself, + // so an open conflict on a large note cost hundreds of megabytes per sync. + assertThat(java.util.Collections.frequency(backend.events, "readAttachment")).isEqualTo(1); + assertThat(java.util.Collections.frequency(store.events, "writeAttachment")).isEqualTo(1); + } + private static SyncSnapshot snapshot(SyncRecord... records) { return new SyncSnapshot(Arrays.asList(records)); } @@ -402,6 +458,10 @@ private static final class FakeStore implements SyncStore { private List appliedConflicts = Collections.emptyList(); private SyncState state = SyncState.idle(); private final Map attachments = new HashMap<>(); + + /** Blobs written through writeAttachment: the durable cache RoomSyncStore keeps. */ + private final java.util.Set durable = new java.util.HashSet<>(); + private Collection attachmentHashes = Collections.emptyList(); private final List states = new ArrayList<>(); private final List events = new ArrayList<>(); @@ -458,6 +518,12 @@ public void writeAttachment(String sha256, long sizeBytes, InputStream content) throws IOException { events.add("writeAttachment"); attachments.put(sha256, readAll(content)); + durable.add(sha256); + } + + @Override + public boolean hasDurableAttachment(String sha256, long sizeBytes) { + return durable.contains(sha256); } @Override @@ -486,6 +552,11 @@ private static final class FakeBackend implements SyncBackend { private IOException readFailure; private int writeSnapshotCalls; + /** When set, the remote read announces itself and then waits to be released. */ + private java.util.concurrent.CountDownLatch readStarted; + + private java.util.concurrent.CountDownLatch readReleased; + FakeBackend(SyncSnapshot snapshot) { this.snapshot = snapshot; } @@ -496,19 +567,28 @@ public String getIdentifier() { } @Override - public SyncSnapshot readSnapshot() throws IOException { + public RemoteSnapshot readSnapshotResult() throws IOException { // Recorded so a test asserting "the remote was never read" actually proves it. events.add("readSnapshot"); if (readFailure != null) { throw readFailure; } - return snapshot; + if (readStarted != null) { + readStarted.countDown(); + try { + readReleased.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new java.io.InterruptedIOException(); + } + } + return RemoteSnapshot.of(snapshot); } @Override - public void writeSnapshot(SyncSnapshot snapshot) { + public void publish(SyncPublication publication) { events.add("writeSnapshot"); - this.snapshot = snapshot; + this.snapshot = publication.getSnapshot(); writeSnapshotCalls++; } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java new file mode 100644 index 00000000..18165493 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java @@ -0,0 +1,103 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +/** + * The single verifier every attachment transfer goes through. + * + *

Four implementations used to disagree on the details pinned here — one had no size ceiling, + * one returned false where the others threw — so a blob accepted on one path could be refused on + * the next. + */ +public class VerifyingInputStreamTest { + + private static final byte[] BYTES = "attachment bytes".getBytes(StandardCharsets.UTF_8); + private static final String HASH = Sha256.of(BYTES); + + @Test + public void acceptsTheDeclaredBytesAndReportsTheirSize() throws Exception { + long size = + VerifyingInputStream.verify( + new ByteArrayInputStream(BYTES), HASH, (long) BYTES.length); + + assertThat(size).isEqualTo(BYTES.length); + } + + @Test + public void rejectsAChecksumMismatch() { + AttachmentIntegrityException failure = + expectIntegrityFailure(new ByteArrayInputStream(BYTES), Sha256.of("other"), null); + + assertThat(failure).hasMessageThat().contains("checksum"); + } + + @Test + public void rejectsASizeMismatch() { + AttachmentIntegrityException failure = + expectIntegrityFailure(new ByteArrayInputStream(BYTES), HASH, 3L); + + assertThat(failure).hasMessageThat().contains("size does not match"); + } + + @Test + public void rejectsABlobPastTheCeilingAsAnIntegrityFailure() throws Exception { + // A plain IOException here escaped the "skip a corrupt candidate" path in the Drive + // backend and failed the whole sync on one oversized object. + VerifyingInputStream verifying = + new VerifyingInputStream(new ByteArrayInputStream(BYTES), HASH, null, 4L); + try { + verifying.verifyEndOfStream(); + throw new AssertionError("Expected the ceiling to be enforced"); + } catch (AttachmentIntegrityException expected) { + assertThat(expected).hasMessageThat().contains("exceeds the sync size limit"); + } + } + + @Test + public void drainsWhatTheDestinationLeftUnreadBeforeJudging() throws Exception { + // A destination that stopped early must not be able to accept a blob it never finished + // checking: the tail is read here and the mismatch surfaces. + byte[] longer = "attachment bytes plus more".getBytes(StandardCharsets.UTF_8); + VerifyingInputStream verifying = + new VerifyingInputStream(new ByteArrayInputStream(longer), HASH, null); + byte[] buffer = new byte[BYTES.length]; + assertThat(verifying.read(buffer, 0, buffer.length)).isEqualTo(BYTES.length); + + try { + verifying.verifyEndOfStream(); + throw new AssertionError("Expected the drained tail to fail the checksum"); + } catch (AttachmentIntegrityException expected) { + assertThat(verifying.bytesRead()).isEqualTo(longer.length); + } + } + + @Test + public void verifyingTwiceIsANoOp() throws Exception { + VerifyingInputStream verifying = + new VerifyingInputStream( + new ByteArrayInputStream(BYTES), HASH, (long) BYTES.length); + while (verifying.read(new byte[8]) != -1) { + // Reaches end of stream, which verifies once inside the read. + } + + verifying.verifyEndOfStream(); + verifying.verifyEndOfStream(); + } + + private static AttachmentIntegrityException expectIntegrityFailure( + ByteArrayInputStream input, String hash, Long size) { + try { + VerifyingInputStream.verify(input, hash, size); + } catch (AttachmentIntegrityException expected) { + return expected; + } catch (IOException unexpected) { + throw new AssertionError(unexpected); + } + throw new AssertionError("Expected an integrity failure"); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java new file mode 100644 index 00000000..e26908ff --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java @@ -0,0 +1,84 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonObject; +import org.junit.Test; + +/** + * The one walk over an Editor.js document that finds attachment references. + * + *

Three walkers with three rule sets used to exist; the same note could relocate correctly after + * a ZIP restore and show broken links after a Drive restore depending on which one ran. + */ +public class EditorAttachmentBlocksTest { + + private static final String DOCUMENT = + "[{\"id\":\"p\",\"type\":\"paragraph\",\"data\":{\"text\":\"hello there\"}}," + + "{\"id\":\"a\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_5/one.pdf\",\"name\":\"one.pdf\"}}}," + + "{\"id\":\"g\",\"type\":\"gallery\",\"data\":{\"files\":[{\"url\":\"editorjs://attachments/note_5/two.png\"},{\"url\":\"editorjs://attachments/note_5/three.png\"}]}}," + + "{\"id\":\"v\",\"type\":\"video\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_5/four.mp4\"}}}]"; + + @Test + public void rewritesEveryFileUrlWhateverTheBlockType() { + String rewritten = + EditorAttachmentBlocks.rewriteUrls( + DOCUMENT, url -> url.replace("note_5", "note_9")); + + // The old sync walker filtered on the attaches and image types and never looked at + // files[]; the gallery and video blocks stayed pointing at the sending device's files. + assertThat(rewritten).doesNotContain("note_5"); + assertThat(EditorAttachmentBlocks.fileUrls(rewritten)) + .containsExactly( + "editorjs://attachments/note_9/one.pdf", + "editorjs://attachments/note_9/two.png", + "editorjs://attachments/note_9/three.png", + "editorjs://attachments/note_9/four.mp4") + .inOrder(); + // Text is carried verbatim: the paragraph's markup is not HTML-escaped on the way through. + assertThat(rewritten).contains("hello there"); + } + + @Test + public void returnsTheDocumentItselfWhenNothingChanges() { + // Identity, not equality: a note nothing touched must stay byte-identical on every + // device, and a re-serialization would already have been a different string. + assertThat(EditorAttachmentBlocks.rewriteUrls(DOCUMENT, url -> null)) + .isSameInstanceAs(DOCUMENT); + assertThat(EditorAttachmentBlocks.rewriteUrls(DOCUMENT, url -> url)) + .isSameInstanceAs(DOCUMENT); + } + + @Test + public void leavesAnUnreadableDocumentAlone() { + String broken = "[{\"type\":\"attaches\",\"data\":"; + + assertThat(EditorAttachmentBlocks.rewriteUrls(broken, url -> "x")).isSameInstanceAs(broken); + assertThat(EditorAttachmentBlocks.fileUrls(broken)).isEmpty(); + assertThat(EditorAttachmentBlocks.rewriteUrls(null, url -> "x")).isNull(); + } + + @Test + public void rewritingIsIdempotent() { + String once = EditorAttachmentBlocks.rewriteUrls(DOCUMENT, url -> url + "?v=2"); + String again = EditorAttachmentBlocks.rewriteUrls(once, url -> url); + + // Sync hashes the rewritten document on both devices, so re-parsing and re-serializing + // the output has to reproduce it exactly. + assertThat(again).isSameInstanceAs(once); + assertThat(EditorAttachmentBlocks.rewriteUrls(once, url -> url.replace("?v=2", "?v=2"))) + .isSameInstanceAs(once); + } + + @Test + public void findsTheFileOfABlockInEitherShape() { + JsonObject single = EditorAttachmentBlocks.findFile(DOCUMENT, "a"); + JsonObject fromList = EditorAttachmentBlocks.findFile(DOCUMENT, "g"); + + assertThat(single.get("name").getAsString()).isEqualTo("one.pdf"); + assertThat(fromList.get("url").getAsString()) + .isEqualTo("editorjs://attachments/note_5/two.png"); + assertThat(EditorAttachmentBlocks.findFile(DOCUMENT, "p")).isNull(); + assertThat(EditorAttachmentBlocks.findFile(DOCUMENT, "missing")).isNull(); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java index 7b51a176..7d84f0f4 100644 --- a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java @@ -65,6 +65,101 @@ public void rewritesTheEditorBlocksThatCarryTheSameUrls() throws Exception { assertThat(result.valueJson).doesNotContain("note_5"); } + @Test + public void adoptsStagedFilesIntoTheFolderOfTheIdTheNoteKept() throws Exception { + File staging = temporaryFolder.newFolder("staging"); + File staged = stage(staging, 5, "photo.jpg", "from the archive"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 5, 5, attachmentsJson(5, "photo.jpg"), null); + + // Same id, same name: the reference is already right, only the bytes had to move. + assertThat(result.changed).isFalse(); + File adopted = new File(new File(root, "note_5"), "photo.jpg"); + assertThat(contentOf(adopted)).isEqualTo("from the archive"); + assertThat(staged.exists()).isFalse(); + } + + @Test + public void adoptsStagedFilesIntoTheFolderOfAReassignedId() throws Exception { + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 5, "photo.jpg", "from the archive"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 5, 12, attachmentsJson(5, "photo.jpg"), null); + + assertThat(result.changed).isTrue(); + assertThat(result.attachmentsJson).contains("note_12"); + assertThat(contentOf(new File(new File(root, "note_12"), "photo.jpg"))) + .isEqualTo("from the archive"); + assertThat(new File(root, "note_5").exists()).isFalse(); + } + + @Test + public void neverOverwritesAFileAnotherNoteAlreadyShows() throws Exception { + // Extraction used to land straight in the live folder, so a foreign archive replaced the + // bytes of whichever local note shared the row id. The existing file stays; the staged + // one is adopted under a fresh name and the reference follows it. + File mine = seed(7, "photo.jpg", "mine"); + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 7, "photo.jpg", "foreign"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 7, 7, attachmentsJson(7, "photo.jpg"), null); + + assertThat(contentOf(mine)).isEqualTo("mine"); + assertThat(result.changed).isTrue(); + assertThat(result.attachmentsJson).doesNotContain("note_7/photo.jpg"); + File[] files = new File(root, "note_7").listFiles(); + assertThat(files).hasLength(2); + String adoptedName = + files[0].getName().equals("photo.jpg") ? files[1].getName() : files[0].getName(); + assertThat(adoptedName).startsWith("photo-"); + assertThat(adoptedName).endsWith(".jpg"); + assertThat(result.attachmentsJson).contains("note_7/" + adoptedName); + } + + @Test + public void reusesAnIdenticalFileInsteadOfDuplicatingIt() throws Exception { + seed(7, "photo.jpg", "same bytes"); + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 7, "photo.jpg", "same bytes"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 7, 7, attachmentsJson(7, "photo.jpg"), null); + + assertThat(result.changed).isFalse(); + assertThat(new File(root, "note_7").listFiles()).hasLength(1); + } + + @Test + public void fallsBackToTheLiveFolderWhenNothingWasStaged() throws Exception { + // A note restored by a release that extracted into the live folders still relocates. + seed(5, "photo.jpg", "already live"); + File staging = temporaryFolder.newFolder("staging"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 5, 12, attachmentsJson(5, "photo.jpg"), null); + + assertThat(result.changed).isTrue(); + assertThat(contentOf(new File(new File(root, "note_12"), "photo.jpg"))) + .isEqualTo("already live"); + } + + private static File stage(File staging, int noteId, String name, String content) + throws Exception { + File folder = new File(staging, "note_" + noteId); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + File file = new File(folder, name); + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return file; + } + @Test public void leavesReferencesThatBelongToAnotherNoteAlone() throws Exception { seed(7, "other.png", "not mine"); diff --git a/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java b/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java new file mode 100644 index 00000000..fb4d2061 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java @@ -0,0 +1,103 @@ +package com.pasich.mynotes.utils.backup.local; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.Gson; +import com.pasich.mynotes.utils.backup.models.JsonBackup; +import com.pasich.mynotes.utils.constants.Backup; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Reading a backup archive must not be able to change a note that already exists. + * + *

Attachments used to be extracted straight into the live {@code attachments/note_} folders, + * before the JSON was even validated and before the restore had decided whether that id belonged to + * some other note already on the device. + */ +public class ZipBackupHelperTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void extractsIntoTheStagingDirectoryAndNeverIntoTheLiveFolders() throws Exception { + File live = temporaryFolder.newFolder("files", "attachments", "note_7"); + File mine = new File(live, "photo.jpg"); + Files.write(mine.toPath(), "mine".getBytes(StandardCharsets.UTF_8)); + File staging = new File(temporaryFolder.getRoot(), "cache/restore-staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put( + Backup.FILE_NAME_BACKUP, + new Gson().toJson(new JsonBackup()).getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_7/photo.jpg", "foreign".getBytes(StandardCharsets.UTF_8)); + + JsonBackup backup = ZipBackupHelper.readZipBackup(zip(entries), staging); + + assertThat(backup.isError()).isFalse(); + assertThat(new String(Files.readAllBytes(mine.toPath()), StandardCharsets.UTF_8)) + .isEqualTo("mine"); + File staged = new File(new File(staging, "note_7"), "photo.jpg"); + assertThat(new String(Files.readAllBytes(staged.toPath()), StandardCharsets.UTF_8)) + .isEqualTo("foreign"); + } + + @Test + public void skipsEveryEntryThatIsNotOneFileInOneNoteFolder() throws Exception { + // The archive is untrusted. "note_5/../note_6/x" lands in a folder the note's JSON never + // names, and "evil/x" or "note_5/sub/x" are places the cleaner never looks, so they would + // be written once and never reclaimed. + File staging = new File(temporaryFolder.getRoot(), "staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put( + Backup.FILE_NAME_BACKUP, + new Gson().toJson(new JsonBackup()).getBytes(StandardCharsets.UTF_8)); + entries.put( + "attachments/note_5/../note_6/x.jpg", "traversal".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/evil/x.jpg", "foreign folder".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_5/sub/x.jpg", "nested".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/../../databases/notes", "escape".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_5/ok.jpg", "kept".getBytes(StandardCharsets.UTF_8)); + + JsonBackup backup = ZipBackupHelper.readZipBackup(zip(entries), staging); + + assertThat(backup.isError()).isFalse(); + assertThat(new File(staging, "note_5/ok.jpg").isFile()).isTrue(); + assertThat(new File(staging, "note_6/x.jpg").exists()).isFalse(); + assertThat(new File(staging, "evil").exists()).isFalse(); + assertThat(new File(staging, "note_5/sub").exists()).isFalse(); + assertThat(new File(temporaryFolder.getRoot(), "databases").exists()).isFalse(); + assertThat(new File(temporaryFolder.getRoot(), "staging/databases").exists()).isFalse(); + } + + @Test + public void anArchiveWithoutTheJsonIsAnError() throws Exception { + File staging = new File(temporaryFolder.getRoot(), "staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put("attachments/note_5/ok.jpg", "kept".getBytes(StandardCharsets.UTF_8)); + + assertThat(ZipBackupHelper.readZipBackup(zip(entries), staging).isError()).isTrue(); + } + + private static ZipInputStream zip(Map entries) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + return new ZipInputStream(new ByteArrayInputStream(bytes.toByteArray())); + } +} From 1cfa9dbfb797440e76b747236723e9d062211b1b Mon Sep 17 00:00:00 2001 From: pasichDev Date: Sat, 5 Sep 2026 01:03:28 +0300 Subject: [PATCH 2/4] fix(sync): repair the regressions the first round introduced A second verified review of the round-one fixes found that several of them had traded one defect for another: - Changing the wire form of block URLs changed the canonical hash of every note with attachments already on Drive from 2.6.50, so an unchanged note conflicted with itself after upgrade, in about half the cases on every sync. Decoding now recognises a payload in the 2.6.50 shape by proving its attachment ids are the old derivation, and rewrites it to the current shape, so it hashes identically to what the upgraded store builds. A hand-built 2.6.50 payload merges with no conflict; a receiver-shaped one conflicts exactly once and then syncs clean. - Staged ZIP attachments were adopted only for notes that survived the already-present filter, so a note whose row existed but whose files were gone never got them back. Adoption now runs for every note. - The relocator asked the mover twice for the same URL and got two different copies, so the attachments column and the editor blocks named different files and the cleaner deleted one. One answer per URL. - The settings guard ran before the apply transaction; an edit made inside it was still committed over. The compare now happens at commit time and the commit is refused when the live values moved. - The sync path still applied the theme from inside applySnapshot, recreating BackupActivity mid-sync. The theme is applied after the sync finishes. - A single corrupt copy of a blob in one root was returned unread when it was the only candidate there, shadowing a good copy in another root. - Reviving a tombstoned tag bypassed same-name reconciliation. - Pruning deleted a bundle without a publish timestamp immediately and measured grace by another device's clock; it now uses Drive's own createdTime and never deletes a bundle whose age Drive does not report. - A raw NUL character sat inside a string literal as a memo separator. 321 unit tests, 80 instrumentation tests, 0 failures; lint 0 errors. --- .../pasich/mynotes/db/RoomSyncStoreTest.java | 344 ++++++++++++++++++ .../data/sync/AttachmentLogicalIds.java | 47 +++ .../data/sync/GoogleDriveSyncBackend.java | 62 ++-- .../data/sync/GoogleDriveSyncWorker.java | 6 +- .../mynotes/data/sync/LegacyNotePayload.java | 103 ++++++ .../mynotes/data/sync/RoomSyncStore.java | 236 ++++++++---- .../data/sync/SnapshotBuildResult.java | 10 + .../mynotes/data/sync/SyncBundleCodec.java | 29 +- .../data/sync/SyncMutationCoordinator.java | 30 ++ .../pasich/mynotes/data/sync/SyncService.java | 50 ++- .../pasich/mynotes/data/sync/SyncStore.java | 13 + .../attach/NoteAttachmentRelocator.java | 49 ++- .../ui/sync/SyncCoordinatorFactory.java | 4 +- .../ui/view/activity/BackupActivity.java | 13 +- .../utils/backup/local/ZipBackupHelper.java | 30 ++ .../data/sync/GoogleDriveSyncBackendTest.java | 66 +++- .../data/sync/LegacyNotePayloadTest.java | 136 +++++++ .../sync/SyncMutationCoordinatorTest.java | 94 +++++ .../mynotes/data/sync/SyncServiceTest.java | 53 +++ .../attach/NoteAttachmentRelocatorTest.java | 29 ++ .../backup/local/ZipBackupHelperTest.java | 16 + 21 files changed, 1306 insertions(+), 114 deletions(-) create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index 83afa855..31f2ffc2 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -418,6 +418,161 @@ public void aNoteReceivedFromAnotherDeviceHashesIdenticallyWhenRebuiltThere() th } } + // ------------------------------------------------- notes published by 2.6.50 + + @Test + public void aNoteSyncedBy2650HashesIdenticallyAfterTheUpgrade() throws Exception { + // The device that published the note under 2.6.50 upgrades. Its unchanged note, rebuilt + // by the upgraded store, has to be the same version as the one on Drive, or the merge + // reports a conflict against the note itself — and, when the old version won, again on + // every sync until the user edited the note. + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("1700000000000_123.png", "photo.png", bytes); + String localUrl = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, "1700000000000_123.png"); + String blocks = + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + localUrl + + "\",\"name\":\"photo.png\"}}}]"; + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setValueJson(blocks); + db.noteDao().addNote(seeded); + SyncRecord onDrive = + published2650(seeded, localUrl, "photo.png", sha256(bytes), bytes.length); + + SyncRecord rebuilt = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncMergeResult merge = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(rebuilt)), + new SyncSnapshot(Collections.singletonList(onDrive))); + + assertThat(rebuilt.getCanonicalPayloadHash()).isEqualTo(onDrive.getCanonicalPayloadHash()); + assertThat(merge.getConflicts()).isEmpty(); + } + + @Test + public void aNoteReceivedUnder2650ConflictsOnceAfterTheUpgradeAndThenSyncsCleanly() + throws Exception { + // The other device: it received the note under 2.6.50, whose restore wrote the sender's + // hash-less id into its column and its own file name into the blocks. Its rebuilt + // version cannot equal the upgraded remote one, so one conflict is expected; what must + // not happen is the same conflict on every sync afterwards. + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + String senderUrl = "editorjs://attachments/note_77/1700000000000_123.png"; + String stableId = "11111111-1111-4111-8111-111111111111"; + String legacyId = + java.util + .UUID + .nameUUIDFromBytes( + (stableId + "\n0\n" + senderUrl + "\nphoto.png") + .getBytes(StandardCharsets.UTF_8)) + .toString(); + int noteId = seedNote("Shopping", "Milk", null); + File folder = new File(context.getFilesDir(), "attachments/note_" + noteId); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + try (FileOutputStream out = new FileOutputStream(new File(folder, legacyId + "-" + hash))) { + out.write(bytes); + } + String receiverUrl = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, legacyId + "-" + hash); + Note received = db.noteDao().getNoteSync(noteId); + received.setAttachments( + "[{\"url\":\"" + + receiverUrl + + "\",\"name\":\"photo.png\",\"id\":\"" + + legacyId + + "\"}]"); + received.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + receiverUrl + + "\",\"name\":\"photo.png\"}}}]"); + db.noteDao().addNote(received); + Note sender = new Note().create("Shopping", "Milk", 1_000L, ""); + sender.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + senderUrl + + "\",\"name\":\"photo.png\"}}}]"); + SyncRecord onDrive = published2650(sender, senderUrl, "photo.png", hash, bytes.length); + + SyncRecord rebuilt = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncMergeResult first = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(rebuilt)), + new SyncSnapshot(Collections.singletonList(onDrive))); + assertThat(first.getConflicts()).hasSize(1); + store.applySnapshot(first.getMergedSnapshot(), first.getConflicts()); + + SyncRecord afterApply = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncMergeResult second = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(afterApply)), + first.getMergedSnapshot()); + + assertThat(second.getConflicts()).isEmpty(); + assertThat(afterApply.getCanonicalPayloadHash()) + .isEqualTo(first.getMergedSnapshot().getRecords().get(0).getCanonicalPayloadHash()); + } + + /** + * The record 2.6.50 published for {@code note}, read back through today's decoder: blocks + * naming the sender's own file, the id derived without the content hash, and the MIME type + * 2.6.50 detected from the display name. Built by hand because that code is gone. + */ + private static SyncRecord published2650( + Note note, String blockUrl, String displayName, String hash, long size) + throws IOException { + String stableId = "11111111-1111-4111-8111-111111111111"; + String legacyId = + java.util + .UUID + .nameUUIDFromBytes( + (stableId + "\n0\n" + blockUrl + "\n" + displayName) + .getBytes(StandardCharsets.UTF_8)) + .toString(); + JsonObject payload = new com.google.gson.Gson().toJsonTree(note).getAsJsonObject(); + payload.remove("a"); + payload.remove("h"); + JsonObject entry = new JsonObject(); + entry.addProperty("id", legacyId); + entry.addProperty("sha256", hash); + entry.addProperty("mimeType", "image/png"); + entry.addProperty("size", size); + entry.addProperty("path", "attachments/" + hash); + entry.addProperty("displayName", displayName); + com.google.gson.JsonArray manifest = new com.google.gson.JsonArray(); + manifest.add(entry); + payload.add("attachmentsManifest", manifest); + com.google.gson.JsonArray hashes = new com.google.gson.JsonArray(); + hashes.add(hash); + payload.add("attachmentHashes", hashes); + JsonObject names = new JsonObject(); + names.addProperty(legacyId, displayName); + payload.add("attachmentNames", names); + SyncRecord asPublished = + SyncRecord.live( + SyncRecord.Type.NOTE, + stableId, + java.time.Instant.ofEpochMilli(1_000L), + payload); + // Through the bundle codec, which is where an old payload meets today's decoder. + com.pasich.mynotes.data.sync.SyncBundleCodec codec = + new com.pasich.mynotes.data.sync.SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(Collections.singletonList(asPublished)), + java.time.Instant.now()); + return codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, stableId); + } + + // ------------------------------------------------- conflicts and records that moved on // ------------------------------------------------- conflicts and records that moved on @Test @@ -585,6 +740,75 @@ public void applySnapshot_leavesSettingsChangedDuringTheSyncAlone() throws Excep .isGreaterThan(2_000L); } + @Test + public void applySnapshot_leavesSettingsChangedInsideTheApplyTransactionAlone() + throws Exception { + // The guard used to run once before the transaction. Applying many notes takes seconds; + // a setting toggled in that window was still overwritten and its digest recorded as the + // baseline, so the next build saw nothing to publish. + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = + new RoomSyncStore( + context, + db, + adapter.helper, + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage::resolve, + file -> sha256(readAll(new java.io.FileInputStream(file))), + record -> { + if (record.getType() == SyncRecord.Type.NOTE) { + // The user flips a setting while the notes are being applied. + adapter.current.set(preferencesWithTheme(2)); + } + }); + preferencesStore.readState(); + adapter.current.set(preferencesWithTheme(1)); + int noteId = seedNote("Applied first", "body", null); + SyncRecord note = onlyNote(preferencesStore.readSnapshot()); + db.syncMetadataDao().setVersion(SyncMetadata.RECORD_TYPE_PREFERENCES, 0, 1_000L, null); + JsonObject changed = note.getPayload(); + changed.addProperty("b", "Remote title"); + SyncRecord remoteNote = + SyncRecord.live( + SyncRecord.Type.NOTE, + note.getId(), + java.time.Instant.ofEpochMilli(2_000L), + changed); + SyncRecord remotePreferences = + SyncRecord.live( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000", + java.time.Instant.ofEpochMilli(2_000L), + new com.google.gson.Gson() + .toJsonTree(preferencesWithTheme(3)) + .getAsJsonObject()); + + preferencesStore.applySnapshot( + new SyncSnapshot(java.util.Arrays.asList(remoteNote, remotePreferences)), + Collections.emptyList(), + SyncState.success("google-drive", java.time.Instant.now(), 0)); + + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Remote title"); + assertThat(adapter.committed.get()).isNull(); + assertThat(adapter.current.get().getThemeValue()).isEqualTo(2); + // Not recorded as holding the remote version, and the final state still lands. + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000") + .updatedAt) + .isEqualTo(1_000L); + assertThat(preferencesStore.readState().getStatus()).isEqualTo(SyncState.Status.SUCCESS); + preferencesStore.buildSnapshot(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000") + .updatedAt) + .isGreaterThan(2_000L); + } + @Test public void applySnapshot_skipsAnUnusablePreferencesPayloadInsteadOfFailingEverySync() throws Exception { @@ -741,6 +965,92 @@ public void applySnapshot_keepsTheLocalTagWhenItHoldsTheWinningIdentity() throws .isNull(); } + @Test + public void applySnapshot_doesNotReviveADeletedTagBesideItsSameNamedSuccessor() + throws Exception { + // Device A deleted "Work" and created a new "Work"; device B edited the old one later. + // Reviving the old row put a second "Work" beside the new one. The local identity holds + // the smaller id, so the old one is tombstoned afresh and retires everywhere next sync. + long newId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + newId, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + 2_000L, + null)); + long oldId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + oldId, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + 1_000L, + null)); + db.tagsDao().deleteById(oldId); + db.syncMetadataDao().markDeleted(SyncMetadata.RECORD_TYPE_TAG, oldId, 1_500L); + SyncRecord editedElsewhere = + SyncRecord.live( + SyncRecord.Type.TAG, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + java.time.Instant.ofEpochMilli(3_000L), + remoteTag("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "Work").getPayload()); + + store.applySnapshot( + new SyncSnapshot(Collections.singletonList(editedElsewhere)), + Collections.emptyList()); + + assertThat(tagsNamed("Work")).isEqualTo(1); + assertThat(db.tagsDao().getTagSync(newId)).isNotNull(); + SyncMetadataEntity old = db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, oldId); + assertThat(old.deletedAt).isNotNull(); + assertThat(old.updatedAt).isGreaterThan(3_000L); + } + + @Test + public void applySnapshot_revivesADeletedTagAndRetiresItsSameNamedSuccessorWhenItWins() + throws Exception { + long newId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + newId, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + 2_000L, + null)); + long oldId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + oldId, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + 1_000L, + null)); + db.tagsDao().deleteById(oldId); + db.syncMetadataDao().markDeleted(SyncMetadata.RECORD_TYPE_TAG, oldId, 1_500L); + SyncRecord editedElsewhere = + SyncRecord.live( + SyncRecord.Type.TAG, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + java.time.Instant.ofEpochMilli(3_000L), + remoteTag("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "Work").getPayload()); + + store.applySnapshot( + new SyncSnapshot(Collections.singletonList(editedElsewhere)), + Collections.emptyList()); + + assertThat(tagsNamed("Work")).isEqualTo(1); + assertThat(db.tagsDao().getTagSync(oldId)).isNotNull(); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, oldId).deletedAt) + .isNull(); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, newId).deletedAt) + .isNotNull(); + } + private static SyncRecord remoteTag(String stableId, String name) { JsonObject payload = new JsonObject(); payload.addProperty("b", name); @@ -823,6 +1133,40 @@ public void buildSnapshot_describesAMissingFileFromWhatTheColumnRemembersAndRest assertThat(readAll(new java.io.FileInputStream(repaired))).isEqualTo(bytes); } + @Test + public void buildSnapshot_stillNamesTheNoteWhenARememberedBlobIsNowhereToBeFound() + throws Exception { + // Published from the remembered hash and size, the note is fine as long as the cache or + // Drive holds the bytes. When neither does — a different account, say — the service has + // to be able to name the note, not fail forever on a hash. + byte[] bytes = "gone everywhere".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + String logicalId = "7d444840-9dc0-11d1-b245-5ffdce74fad2"; + int noteId = seedNote("Shopping list", "body", null); + Note note = db.noteDao().getNoteSync(noteId); + note.setAttachments( + "[{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, logicalId + "-" + hash) + + "\",\"name\":\"photo.png\",\"id\":\"" + + logicalId + + "\",\"sha256\":\"" + + hash + + "\",\"size\":" + + bytes.length + + ",\"mimeType\":\"image/png\"}]"); + db.noteDao().addNote(note); + + SnapshotBuildResult result = store.buildSnapshot(); + + assertThat(result.isPublishable()).isTrue(); + assertThat(store.hasAttachment(hash)).isFalse(); + SnapshotProblem problem = store.describeMissingAttachment(hash); + assertThat(problem).isNotNull(); + assertThat(problem.getKind()).isEqualTo(SnapshotProblem.Kind.MISSING_ATTACHMENT); + assertThat(problem.getLabel()).isEqualTo("Shopping list"); + } + @Test public void buildSnapshot_namesTheNoteWhoseAttachmentCannotBeFound() { int noteId = seedNote("Shopping list", "body", null); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java new file mode 100644 index 00000000..ef9ab7ed --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java @@ -0,0 +1,47 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +/** + * How an attachment that the editor never gave an id receives one. + * + *

Existing editor data predates logical attachment ids, so the store derives one from the note's + * stable id, the attachment's position, its stored URL and its display name. Two releases spell + * this differently: 2.6.50 stopped there, and its ids can therefore describe two different blobs at + * once, which failed every publish for an account whose note had a replaced attachment. Later + * releases fold in the content hash. Both spellings live here because the bundle decoder has to + * recognise the old one to upgrade a 2.6.50 payload into the current shape. + */ +final class AttachmentLogicalIds { + + private AttachmentLogicalIds() {} + + /** The current derivation: one id per (note, position, URL, name, content). */ + @NonNull + static String derive( + @NonNull String noteStableId, + int index, + @NonNull String url, + @NonNull String displayName, + @NonNull String sha256) { + return nameUuid( + noteStableId + "\n" + index + "\n" + url + "\n" + displayName + "\n" + sha256); + } + + /** The 2.6.50 derivation, kept only so its payloads can be recognised. */ + @NonNull + static String deriveLegacy( + @NonNull String noteStableId, + int index, + @NonNull String url, + @NonNull String displayName) { + return nameUuid(noteStableId + "\n" + index + "\n" + url + "\n" + displayName); + } + + @NonNull + private static String nameUuid(@NonNull String source) { + return UUID.nameUUIDFromBytes(source.getBytes(StandardCharsets.UTF_8)).toString(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index f130a1b4..dbf888ec 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -307,8 +307,12 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO private void pruneSupersededBundles(@NonNull Collection frontierBundleIds) { long cutoff = clock.millis() - BUNDLE_PRUNE_GRACE_MILLIS; for (BundleFile bundle : lastReadBundles) { + // The age comes from Drive's own creation time, never from a property the publishing + // device stamped with its clock; a bundle whose age Drive does not report is left + // alone rather than assumed old. if (frontierBundleIds.contains(bundle.logicalId) - || (bundle.publishedAtMillis != null && bundle.publishedAtMillis > cutoff)) { + || bundle.createdAtMillis == null + || bundle.createdAtMillis > cutoff) { continue; } try { @@ -347,26 +351,33 @@ public synchronized boolean hasAttachment(@NonNull String sha256) throws IOExcep @Nullable @Override public synchronized InputStream readAttachment(@NonNull String sha256) throws IOException { - for (String folderId : findFolderIds()) { + List roots = findFolderIds(); + Map> candidatesByRoot = new java.util.LinkedHashMap<>(); + int total = 0; + for (String folderId : roots) { List candidates = listAttachmentCandidates(folderId, sha256); - if (candidates.isEmpty()) { - continue; - } - String chosen = null; + candidatesByRoot.put(folderId, candidates); + total += candidates.size(); for (AttachmentCandidate candidate : candidates) { if (isVerifiedWithoutReading(candidate, sha256, null)) { - chosen = candidate.id; - break; + return openAttachment(candidate.id); } } - if (chosen == null && candidates.size() == 1) { - // A single corrupt copy fails the caller's verifier exactly as it would fail - // one here; the difference is one download instead of two. - chosen = candidates.get(0).id; - } - if (chosen == null) { - chosen = findVerifiedAttachment(folderId, sha256, null); + } + if (total == 1) { + // The only copy in the whole account: a corrupt one fails the caller's verifier + // exactly as it would fail one here, and there is nothing else to fall back to. The + // difference is one download instead of two. + for (List candidates : candidatesByRoot.values()) { + if (!candidates.isEmpty()) { + return openAttachment(candidates.get(0).id); + } } + } + // Several unverified copies, across duplicate roots or within one: read ahead of time so + // a corrupt copy in one root cannot shadow the good copy in another. + for (String folderId : roots) { + String chosen = findVerifiedAttachment(folderId, sha256, null); if (chosen != null) { return openAttachment(chosen); } @@ -626,20 +637,20 @@ private static List computeFrontier( private static final class BundleFile { private final String fileId; @Nullable private final String logicalId; - @Nullable private final Long publishedAtMillis; + @Nullable private final Long createdAtMillis; private BundleFile( @NonNull String fileId, @Nullable String logicalId, - @Nullable Long publishedAtMillis) { + @Nullable Long createdAtMillis) { this.fileId = fileId; this.logicalId = logicalId; - this.publishedAtMillis = publishedAtMillis; + this.createdAtMillis = createdAtMillis; } @NonNull private BundleFile withLogicalId(@NonNull String id) { - return new BundleFile(fileId, id, publishedAtMillis); + return new BundleFile(fileId, id, createdAtMillis); } } @@ -648,7 +659,7 @@ private List findBundles(@NonNull String folderId) throws IOExceptio JsonArray bundles = listFiles( ownedFilesQuery(folderId, PROPERTY_BUNDLE, "1"), - "files(id,name,appProperties)"); + "files(id,name,createdTime)"); List result = new ArrayList<>(bundles.size()); for (int index = 0; index < bundles.size(); index++) { JsonObject file = bundles.get(index).getAsJsonObject(); @@ -656,19 +667,20 @@ private List findBundles(@NonNull String folderId) throws IOExceptio new BundleFile( file.get("id").getAsString(), null, - publishedAtOf(file.getAsJsonObject("appProperties")))); + createdAtOf(optionalString(file, "createdTime")))); } result.sort(Comparator.comparing(file -> file.fileId)); return result; } + /** Drive reports {@code createdTime} as RFC 3339; anything else counts as unknown. */ @Nullable - private static Long publishedAtOf(@Nullable JsonObject appProperties) { - if (appProperties == null || !appProperties.has(PROPERTY_BUNDLE_PUBLISHED_AT)) { + private static Long createdAtOf(@Nullable String createdTime) { + if (createdTime == null) { return null; } try { - return Long.parseLong(appProperties.get(PROPERTY_BUNDLE_PUBLISHED_AT).getAsString()); + return java.time.Instant.parse(createdTime).toEpochMilli(); } catch (RuntimeException malformed) { return null; } @@ -763,7 +775,7 @@ private boolean isVerifiedWithoutReading( @NonNull private static String memoKey(@NonNull String candidateId, @NonNull String sha256) { - return candidateId + "" + sha256; + return candidateId + " " + sha256; } /** diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java index 00d83bac..9b4003b3 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java @@ -70,7 +70,11 @@ public Result doWork() { getApplicationContext(), dependencies.database(), dependencies.preferenceHelper())) - .sync(new GoogleDriveSyncBackend(authorization.getAccessToken())); + .sync( + new GoogleDriveSyncBackend(authorization.getAccessToken()), + // Re-checked under the sync lock: a disconnect during the + // token round trip must not be followed by a write. + () -> dependencies.preferenceHelper().isSyncEnabled()); if (state.getStatus() == SyncState.Status.SUCCESS) return Result.success(); return isRetryable(state.getErrorMessage()) ? Result.retry() : Result.failure(); } catch (Exception error) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java b/app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java new file mode 100644 index 00000000..6facbaeb --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java @@ -0,0 +1,103 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.pasich.mynotes.extendedEditor.attach.AttachmentUrl; +import com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks; +import java.util.ArrayList; +import java.util.List; + +/** + * Brings a note published by 2.6.50 into the shape the current store builds. + * + *

2.6.50 published a note's editor blocks with the writing device's own file URLs and derived + * attachment ids without the content hash. The current store publishes wire references and ids that + * include the hash, so an unchanged note read back from Drive after the upgrade hashed differently + * from the same note rebuilt locally at the same timestamp. The merge reported a conflict against + * itself — and, when the old version won the tiebreaker, applied it, rebuilt it in the new shape, + * found nothing to publish because the merged snapshot equalled the remote one, and raised the same + * conflict again on every sync until the user happened to edit the note. + * + *

The old shape carries everything the new derivation needs: the note's stable id, each block's + * URL, each entry's name and hash. A payload is upgraded only when every manifest id is provably + * the old derivation of exactly those inputs, so a payload written by any other rule is left alone. + * + *

Deliberately free of {@code android.*}: the equality it restores is what every sync depends + * on. + */ +final class LegacyNotePayload { + + private LegacyNotePayload() {} + + /** + * Rewrites {@code payload} in place when it is a 2.6.50-shaped note. + * + * @return true when the payload was upgraded. + */ + static boolean upgrade(@NonNull String noteStableId, @NonNull JsonObject payload) { + JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); + JsonElement blocks = payload.get("f"); + if (manifest == null + || manifest.size() == 0 + || blocks == null + || !blocks.isJsonPrimitive() + || !blocks.getAsJsonPrimitive().isString()) { + return false; + } + List urls = EditorAttachmentBlocks.fileUrls(blocks.getAsString()); + if (urls.size() != manifest.size()) { + return false; + } + List newIds = new ArrayList<>(manifest.size()); + for (int index = 0; index < manifest.size(); index++) { + JsonElement element = manifest.get(index); + if (!element.isJsonObject()) { + return false; + } + JsonObject entry = element.getAsJsonObject(); + String url = urls.get(index); + if (AttachmentUrl.parse(url) == null + || !isString(entry.get("id")) + || !isString(entry.get("sha256")) + || !isString(entry.get("displayName"))) { + return false; + } + String displayName = entry.get("displayName").getAsString(); + if (!entry.get("id") + .getAsString() + .equals( + AttachmentLogicalIds.deriveLegacy( + noteStableId, index, url, displayName))) { + return false; + } + newIds.add( + AttachmentLogicalIds.derive( + noteStableId, + index, + url, + displayName, + entry.get("sha256").getAsString())); + } + + JsonObject names = new JsonObject(); + for (int index = 0; index < manifest.size(); index++) { + JsonObject entry = manifest.get(index).getAsJsonObject(); + entry.addProperty("id", newIds.get(index)); + names.addProperty(newIds.get(index), entry.get("displayName").getAsString()); + } + payload.add("attachmentNames", names); + int[] position = {0}; + payload.addProperty( + "f", + EditorAttachmentBlocks.rewriteUrls( + blocks.getAsString(), + url -> AttachmentWireUrl.forLogicalId(newIds.get(position[0]++)))); + return true; + } + + private static boolean isString(JsonElement value) { + return value != null && value.isJsonPrimitive() && value.getAsJsonPrimitive().isString(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 6f113cb1..4885eadd 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -32,7 +32,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URLConnection; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -75,6 +74,13 @@ public final class RoomSyncStore implements SyncStore { /** Whether every note folder has been indexed into {@link #localAttachments}. */ private volatile boolean noteFoldersIndexed; + /** + * Attachments the last build described from the column alone because their file is gone, keyed + * by hash, so the service can name the note when no endpoint holds the blob either. + */ + private final Map rememberedOnlyAttachments = + new ConcurrentHashMap<>(); + /** * Set when an apply actually changed the visible settings, so the screen can redraw. * @@ -180,6 +186,7 @@ public SnapshotBuildResult buildSnapshot() throws IOException { ensureSeeded(); List records = new ArrayList<>(); List problems = new ArrayList<>(); + rememberedOnlyAttachments.clear(); try { for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { if (metadata.deletedAt != null) { @@ -237,27 +244,18 @@ private void applySnapshotInternal( @Nullable SyncState finalState) throws IOException { PreferencesBackup stagedPreferences = selectedPreferences(snapshot); - if (stagedPreferences != null && preferencesChangedSinceBuild()) { - // The settings screens write SharedPreferences directly and nothing touches the sync - // record for them, so the stale-record guard below cannot protect a setting changed - // while the sync was in flight. Committing the merged version would overwrite it and - // record its digest as the baseline, hiding the loss from the next build. Leaving the - // live values alone means the next build sees them differ from the baseline and - // publishes them as the local edit they are. - Log.w(TAG, "Skipping synchronized preferences; they were edited during this sync"); - stagedPreferences = null; - } String stagedPreferencesJson = stagedPreferences == null ? null : gson.toJson(stagedPreferences); String stagedPreferencesTarget = stagedPreferences == null ? "" : preferencesDigest(stagedPreferences); - String preferencesBaseline = stagedPreferences == null ? "" : livePreferencesDigest(); SyncRecord preferencesRecord = snapshot.find(SyncRecord.Type.PREFERENCES, PREFERENCES_STABLE_ID); long stagedPreferencesUpdatedAt = preferencesRecord == null ? 0L : preferencesRecord.getUpdatedAt().toEpochMilli(); boolean deferFinalState = stagedPreferences != null && finalState != null; - boolean applyPreferences = stagedPreferences != null; + // The live digest at the moment the journal is written, decided inside the transaction + // below; the commit afterwards refuses to run if the settings moved again since. + String[] preferencesBaseline = {null}; try { database.runInTransaction( () -> { @@ -271,6 +269,7 @@ private void applySnapshotInternal( // conflict for one of them names versions that no longer describe // the local record, so it must not be stored for the user to apply. Set skippedKeys = new HashSet<>(); + SyncMetadataEntity[] preferencesMetadata = {null}; for (SyncRecord record : snapshot.getRecords()) { String key = recordKey(record); SyncMetadataEntity metadata = byStableId.get(key); @@ -292,12 +291,15 @@ private void applySnapshotInternal( } if (metadata == null) continue; if (record.getType() == SyncRecord.Type.PREFERENCES - && !record.isTombstone() - && !applyPreferences) { - // Invalid payload or edited mid-sync: the version is not - // going to be committed, so it must not be recorded as the - // one this device holds either. - skippedKeys.add(key); + && !record.isTombstone()) { + // Decided last, once every other record has been applied: + // the settings screens write SharedPreferences directly and + // nothing touches the sync record for them, so the + // stale-record guard below cannot see a setting changed while + // this transaction ran. Committing the merged version anyway + // overwrote such an edit and recorded its digest as the + // baseline, hiding the loss from the next build. + preferencesMetadata[0] = metadata; continue; } // The snapshot was built before Drive was read and every blob @@ -327,7 +329,10 @@ private void applySnapshotInternal( transactionFailureInjector.afterRecordApplied(record); continue; } - applyPayload(metadata, record.getPayload()); + if (!applyPayload(metadata, record.getPayload())) { + skippedKeys.add(key); + continue; + } database.syncMetadataDao() .setVersion( metadata.recordType, @@ -336,20 +341,49 @@ private void applySnapshotInternal( null); transactionFailureInjector.afterRecordApplied(record); } - persistConflicts(conflicts, skippedKeys); - if (stagedPreferencesJson != null) { - database.syncPendingPreferencesDao() - .upsert( - new SyncPendingPreferencesEntity( - 1, - stagedPreferencesJson, - stagedPreferencesTarget, - preferencesBaseline, - stagedPreferencesUpdatedAt, - false, - 0L, - "")); + if (preferencesMetadata[0] != null) { + String preferencesKey = recordKey(preferencesMetadata[0]); + String live = livePreferencesDigest(); + String baseline = preferences.getString(PREFERENCES_HASH, null); + boolean edited = baseline != null && !baseline.equals(live); + boolean stale = + preferencesMetadata[0].updatedAt + > stagedPreferencesUpdatedAt; + if (stagedPreferencesJson == null || edited || stale) { + // Unusable payload, or edited since the build: the version + // is not going to be committed, so it is not recorded as + // the one this device holds either. The next build sees the + // live values differ from the baseline and publishes them + // as the local edit they are. + if (edited) { + Log.w( + TAG, + "Skipping synchronized preferences; they were" + + " edited during this sync"); + } + skippedKeys.add(preferencesKey); + } else { + preferencesBaseline[0] = live; + database.syncMetadataDao() + .setVersion( + preferencesMetadata[0].recordType, + preferencesMetadata[0].localId, + stagedPreferencesUpdatedAt, + null); + database.syncPendingPreferencesDao() + .upsert( + new SyncPendingPreferencesEntity( + 1, + stagedPreferencesJson, + stagedPreferencesTarget, + live, + stagedPreferencesUpdatedAt, + false, + 0L, + "")); + } } + persistConflicts(conflicts, skippedKeys); if (finalState != null && !deferFinalState) { database.syncStateDao().upsert(toEntity(finalState)); } @@ -360,11 +394,16 @@ private void applySnapshotInternal( } catch (SyncRuntimeException error) { throw error.ioException; } - if (stagedPreferences != null) { - // The journal is only dropped once the adapter reports a durable commit; a failure - // here leaves it in place for recoverPendingPreferences and keeps the sync state - // retryable rather than claiming success. - commitPendingPreferences(stagedPreferences, stagedPreferencesTarget); + if (deferFinalState || preferencesBaseline[0] != null) { + if (preferencesBaseline[0] != null) { + // The journal is only dropped once the adapter reports a durable commit; a + // failure here leaves it in place for recoverPendingPreferences and keeps the + // sync state retryable rather than claiming success. A commit refused because + // the settings moved in the meantime is not a failure: the journal is dropped + // and the edit is published by the next build. + commitPendingPreferences( + stagedPreferences, stagedPreferencesTarget, preferencesBaseline[0]); + } database.runInTransaction( () -> { database.syncPendingPreferencesDao().clear(); @@ -385,17 +424,6 @@ private static String recordKey(@NonNull SyncRecord record) { return record.getType().getWireValue() + ":" + record.getId(); } - /** - * Whether the live settings differ from what the snapshot was built from. - * - *

Every build records the live digest as the baseline, so a baseline that no longer matches - * means the user changed a setting after the build and before this apply. - */ - private boolean preferencesChangedSinceBuild() { - String baseline = preferences.getString(PREFERENCES_HASH, null); - return baseline != null && !baseline.equals(livePreferencesDigest()); - } - /** * Drops cached blobs nothing can still need. * @@ -548,7 +576,8 @@ private void recoverPendingPreferences() throws IOException { return; case REPLAY: default: - commitPendingPreferences(backup, target); + // Replay was decided because the live values still match the baseline. + commitPendingPreferences(backup, target, pending.baselineHash); finishJournal(pending); } } @@ -566,10 +595,20 @@ private void finishJournal(@NonNull SyncPendingPreferencesEntity pending) { * Applies one journaled preferences payload, failing loudly when it is not durable. * * @param expectedDigest digest the live preferences must show afterwards. + * @param baselineDigest digest the live preferences showed when the write was decided; if they + * no longer match, the user changed a setting in between and the write is refused. + * @return true when the values were written, false when the write was refused as superseded. */ - private void commitPendingPreferences( - @NonNull PreferencesBackup backup, @NonNull String expectedDigest) throws IOException { + private boolean commitPendingPreferences( + @NonNull PreferencesBackup backup, + @NonNull String expectedDigest, + @NonNull String baselineDigest) + throws IOException { String before = livePreferencesDigest(); + if (!before.equals(baselineDigest) && !before.equals(expectedDigest)) { + Log.w(TAG, "Refusing to commit synchronized preferences; they changed meanwhile"); + return false; + } boolean committed; try { committed = preferenceHelper.commitListPreferences(backup); @@ -585,6 +624,7 @@ private void commitPendingPreferences( // The digest doubles as the snapshot-build baseline, so recording it here keeps the next // build from treating a freshly received version as a local edit. preferences.edit().putString(PREFERENCES_HASH, expectedDigest).commit(); + return true; } /** @@ -691,7 +731,14 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { return result; } - private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throws IOException { + /** + * Writes one live version over the local row. + * + * @return false when the version was deliberately not applied and the record's local version + * must therefore not be advanced to it. + */ + private boolean applyPayload(SyncMetadataEntity metadata, JsonObject payload) + throws IOException { // A record that was deleted here and then edited on another device has no row left to // update; @Update on a missing row is a silent no-op, and the tombstone was cleared // regardless, so tasks, categories and tags marked live never came back. The REPLACE @@ -730,14 +777,53 @@ private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throw Tag tag = gson.fromJson(payload, Tag.class); tag.id = metadata.localId; if (revive || database.tagsDao().getTagSync(tag.id) == null) { - database.tagsDao().addTag(tag); - } else { - database.tagsDao().updateTag(tag); + return reviveTag(metadata, tag); } + database.tagsDao().updateTag(tag); } else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { // SharedPreferences is outside Room. applySnapshotInternal journals and commits this // payload only after the Room transaction succeeds. } + return true; + } + + /** + * Puts a deleted tag's row back, unless a tag of that name has since been created here. + * + *

A tag is identified by its name: a note stores the name and the table has no unique index + * on it, so putting the row back beside a same-named one shows the user the same tag twice. The + * same rule as {@link #insertRemoteTag} settles which identity survives: the smaller stable id. + * When the local tag holds it, the reviving identity is tombstoned afresh so that the + * tombstone, now newer than the remote edit, retires it everywhere at the next sync — and the + * record is reported as not applied, so its version is not advanced. + */ + private boolean reviveTag(@NonNull SyncMetadataEntity metadata, @NonNull Tag tag) { + String name = tag.getNameTag(); + Tag existing = + name == null || name.isEmpty() ? null : database.tagsDao().getTagByNameSync(name); + if (existing != null && existing.getId() != tag.id) { + SyncMetadataEntity existingMetadata = + database.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, existing.getId()); + if (existingMetadata != null + && existingMetadata.stableId.compareTo(metadata.stableId) < 0) { + database.syncMetadataDao() + .markDeleted( + SyncMetadata.RECORD_TYPE_TAG, + metadata.localId, + System.currentTimeMillis()); + return false; + } + database.tagsDao().deleteById(existing.getId()); + if (existingMetadata != null) { + database.syncMetadataDao() + .markDeleted( + SyncMetadata.RECORD_TYPE_TAG, + existing.getId(), + System.currentTimeMillis()); + } + } + database.tagsDao().addTag(tag); + return true; } private long insertRemoteRecord(SyncRecord record) throws IOException { @@ -995,8 +1081,11 @@ private void resolvePreferencesConflict( }); // Throws when the write is not durable, leaving the journal in place and the conflict - // unresolved so the user can try again. - commitPendingPreferences(chosen, target); + // unresolved so the user can try again. A setting changed while the choice was being + // applied is treated the same way; recovery then discards the journal as stale. + if (!commitPendingPreferences(chosen, target, baseline)) { + throw new IOException("Settings changed while the conflict was being resolved"); + } try { finalizeResolvedPreferencesConflict(conflictId, resolution.name()); @@ -1273,6 +1362,12 @@ public boolean hasAttachment(@NonNull String sha256) { return resolveLocalAttachment(sha256) != null; } + @Nullable + @Override + public SnapshotProblem describeMissingAttachment(@NonNull String sha256) { + return rememberedOnlyAttachments.get(sha256); + } + @Override public boolean hasDurableAttachment(@NonNull String sha256, long sizeBytes) { File cached = attachmentFile(sha256); @@ -1472,6 +1567,13 @@ private boolean addAttachmentMetadata( file = null; hash = rememberedHash; size = rememberedSize; + rememberedOnlyAttachments.put( + hash, + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + metadata.recordType, + metadata.stableId, + noteTitle)); } else { if (!file.canRead()) { addSnapshotProblem( @@ -1511,18 +1613,12 @@ private boolean addAttachmentMetadata( // otherwise non-canonical id used to pass here and then throw during encode, // failing every publish for the whole account while that note existed. logicalId = - UUID.nameUUIDFromBytes( - (metadata.stableId - + "\n" - + attachmentIndex - + "\n" - + attachment.url - + "\n" - + displayName - + "\n" - + hash) - .getBytes(StandardCharsets.UTF_8)) - .toString(); + AttachmentLogicalIds.derive( + metadata.stableId, + attachmentIndex, + attachment.url, + displayName, + hash); } hashes.add(hash); names.addProperty(logicalId, displayName); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java index 25dd77e0..ddacf042 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java @@ -56,6 +56,16 @@ public SyncSnapshot requireSnapshot() throws IOException { return snapshot; } + /** + * The failure for a problem discovered after the build — a blob the build described from + * remembered metadata that no endpoint turned out to hold — worded like a build failure so the + * user reads the same "which note" message. + */ + @NonNull + public static SnapshotBuildException incompleteBecause(@NonNull SnapshotProblem problem) { + return new SnapshotBuildException(Collections.singletonList(problem)); + } + /** Typed, coarse error suitable for persisted sync state and telemetry. */ public static final class SnapshotBuildException extends IOException { @NonNull private final List problems; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 2c47218a..285aea71 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -36,7 +36,14 @@ public final class SyncBundleCodec { public static final String BUNDLE_FORMAT = "mynotes-sync"; public static final int SCHEMA_VERSION = 1; - /** Wire field mapping a re-keyed manifest id back to the record's own attachment id. */ + /** + * Wire field mapping a re-keyed manifest id back to the record's own attachment id. + * + *

Unknown to 2.6.50, which reads the re-keyed id as the attachment's own; on such a client + * the alternative's attachment identity is lost until it upgrades. Accepted: the field only + * appears when one note's versions disagree about an id's content, and the alternative is still + * restorable there, under the re-keyed id. + */ static final String FIELD_ATTACHMENT_ID_ALIASES = "attachmentIdAliases"; private static final Pattern SHA_256 = Pattern.compile("[0-9a-f]{64}"); @@ -431,7 +438,7 @@ private static void parseLiveRecords( // Bundles written before the device-local fields were stripped still carry them. SyncMetadata.stripDeviceLocalFields(type.getWireValue(), payload); if (type == SyncRecord.Type.NOTE) { - hydrateNoteAttachments(payload, attachmentsById); + hydrateNoteAttachments(id, payload, attachmentsById); } String identity = type.getWireValue() + ":" + id; if (!identities.add(identity)) { @@ -441,8 +448,17 @@ private static void parseLiveRecords( } } + /** + * Rebuilds the local attachment fields from the wire references. + * + *

A payload written by 2.6.50 comes out in the current shape as well, so an unchanged note + * synced before the upgrade hashes exactly as the upgraded store rebuilds it; see {@link + * LegacyNotePayload}. + */ private static void hydrateNoteAttachments( - JsonObject payload, Map attachmentsById) + String noteStableId, + JsonObject payload, + Map attachmentsById) throws IOException { JsonArray attachmentIds = payload.getAsJsonArray("attachmentIds"); JsonObject attachmentNames = payload.getAsJsonObject("attachmentNames"); @@ -476,7 +492,9 @@ private static void hydrateNoteAttachments( : wireId; JsonObject value = attachment.withId(attachmentId).toJson(true); if (attachmentNames != null && attachmentNames.has(wireId)) { - value.addProperty("displayName", attachmentNames.get(wireId).getAsString()); + // Trimmed here as everywhere else: the validator judged the trimmed name, and + // this is the copy the store hashes and shows. + value.addProperty("displayName", attachmentNames.get(wireId).getAsString().trim()); } manifest.add(value); attachmentHashes.add(attachment.sha256); @@ -489,6 +507,7 @@ private static void hydrateNoteAttachments( // The display name restoreAttachments actually uses travels on the manifest entry above; // this map exists only so the payload matches the one the local store builds. payload.add("attachmentNames", namesById); + LegacyNotePayload.upgrade(noteStableId, payload); } @NonNull @@ -523,7 +542,7 @@ private static List parseAlternatives( payload.remove("deletedAt"); SyncMetadata.stripDeviceLocalFields(type.getWireValue(), payload); if (type == SyncRecord.Type.NOTE) { - hydrateNoteAttachments(payload, attachmentsById); + hydrateNoteAttachments(id, payload, attachmentsById); } alternatives.add(SyncRecord.live(type, id, updatedAt, payload)); } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java index c933046b..47f42e9e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java @@ -305,6 +305,7 @@ public void insertNotes(List incoming) { // sat on the restore dialog for tens of seconds doing nothing else. Map existingById = notesByIds(extractNoteIds(incoming)); List notes = withoutNotesAlreadyPresent(incoming, existingById); + adoptAttachmentsOfNotesAlreadyPresent(incoming, notes); if (notes.isEmpty()) return null; long timestamp = resolveBatchTimestamp( @@ -339,6 +340,35 @@ public void insertNotes(List incoming) { }); } + /** + * Gives a note this device already holds the files the archive carries for it. + * + *

A note skipped as already present may still be missing its files — a row restored from a + * JSON backup, or a library whose attachment folder was cleared — and the archive is the only + * place they are. Its staged files are adopted into its own folder like any other restored + * note's; the row is stored again only if a colliding file forced a reference to be rewritten. + */ + private void adoptAttachmentsOfNotesAlreadyPresent( + @NonNull List incoming, @NonNull List inserted) { + Set kept = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + kept.addAll(inserted); + long now = timeProvider.now(); + for (Note note : incoming) { + if (kept.contains(note) || note.getId() <= 0) continue; + if (attachmentRelocation.relocate(note, note.getId())) { + noteDao.updateNoteContent( + note.getId(), + note.getTitle(), + note.getValue(), + note.getValueJson(), + note.getDate(), + note.getTag(), + note.getAttachments()); + touchRecord(SyncMetadata.RECORD_TYPE_NOTE, note.getId(), now); + } + } + } + /** Inserts one group and settles each note's final id, metadata and attachment folder. */ private void assignInsertedNoteIds( @NonNull List notes, long timestamp, @NonNull Map previousIds) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 16e59be9..f53a3db6 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -76,9 +76,26 @@ public static void runWhileNoSyncRuns(@NonNull Runnable action) { } } - /** Runs one serialized manual synchronization attempt and returns its durable final state. */ + /** Runs one serialized synchronization attempt and returns its durable final state. */ @NonNull public SyncState sync(@NonNull SyncBackend backend) { + return sync(backend, () -> true); + } + + /** + * Runs one serialized synchronization attempt, unless {@code stillEnabled} says otherwise once + * the lock is held. + * + *

Disconnect turns sync off and then wipes the account's state under the lock. A worker that + * had already passed its own checks and was waiting for a token could still take the lock after + * that wipe and write the old account's state and conflicts back. The predicate is evaluated + * under the lock, after the wipe has either finished or not yet begun, so a disabled sync never + * gets to write anything. + */ + @NonNull + public SyncState sync( + @NonNull SyncBackend backend, + @NonNull java.util.function.BooleanSupplier stillEnabled) { boolean acquired = false; try { acquired = SYNC_LOCK.tryLock(LOCK_WAIT_SECONDS, TimeUnit.SECONDS); @@ -94,6 +111,13 @@ public SyncState sync(@NonNull SyncBackend backend) { "Another sync is already running; this attempt was temporarily skipped"); } try { + if (!stillEnabled.getAsBoolean()) { + // Deliberately not persisted: there is no account left to record it for. + return SyncState.error( + "google-drive", + safeReadState().getLastSuccessfulSyncAt(), + "Sync was turned off before this attempt could start"); + } return syncExclusively(backend); } finally { SYNC_LOCK.unlock(); @@ -423,7 +447,7 @@ private void synchronizeAttachments( } else { InputStream remoteAttachment = backend.readAttachment(hash); if (remoteAttachment == null) { - throw new IOException("Required attachment is unavailable: " + hash); + throw unavailableAttachment(hash); } copyVerified( hash, expectedSizes.get(hash), remoteAttachment, store::writeAttachment); @@ -470,10 +494,30 @@ private void pinConflictVersion( } } + /** + * The failure for a blob neither endpoint holds. + * + *

When the store published the attachment from remembered metadata — its file gone, the + * bytes expected back from the remote — the message names the note, as a build failure would + * have; a hash alone left the user with a permanently failing sync and no way to find the note. + */ + @NonNull + private IOException unavailableAttachment(@NonNull String hash) { + SnapshotProblem problem = null; + try { + problem = store.describeMissingAttachment(hash); + } catch (RuntimeException ignored) { + // The description is a courtesy; the failure below stands without it. + } + return problem != null + ? SnapshotBuildResult.incompleteBecause(problem) + : new IOException("Required attachment is unavailable: " + hash); + } + private void verifyAttachment(String hash, Long expectedSize, InputStream source) throws IOException { if (source == null) { - throw new IOException("Required attachment is unavailable: " + hash); + throw unavailableAttachment(hash); } // Consume the complete blob before accepting an existing attachment. try (InputStream input = source) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java index 48dac670..71ecc652 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java @@ -1,6 +1,7 @@ package com.pasich.mynotes.data.sync; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.util.Collection; @@ -71,6 +72,18 @@ default java.util.Set getResolvedAlternativeIds() throws IOException { /** True when the complete attachment is locally available. */ boolean hasAttachment(@NonNull String sha256) throws IOException; + /** + * Names the record a blob belongs to when the store published it without holding the bytes. + * + *

A store may describe an attachment from remembered metadata after its file has gone, + * expecting the bytes to come back from the remote. When no endpoint has them either, the + * failure has to name the note to fix, not a hash; a store that never does this answers null. + */ + @Nullable + default SnapshotProblem describeMissingAttachment(@NonNull String sha256) { + return null; + } + /** * True when the blob already sits in the store's own durable cache, as written by {@link * #writeAttachment}, rather than only in a note's folder that a later edit may empty. diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java index de9a5495..8e5eb6af 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java @@ -120,7 +120,16 @@ public static Result adoptStaged( /** Rewrites the column first and, only when it changed, the blocks that mirror it. */ @NonNull private static Result rewrite( - @Nullable String attachmentsJson, @Nullable String valueJson, ReferenceMover mover) { + @Nullable String attachmentsJson, @Nullable String valueJson, ReferenceMover decide) { + // One answer per URL: the column and the blocks name the same files, and a mover that + // picked a fresh name on a collision gave each of them a different copy — the cleaner + // then deleted the one the column did not know and the block rendered a missing file. + java.util.Map> decided = new java.util.HashMap<>(); + ReferenceMover mover = + url -> + decided.computeIfAbsent( + url, key -> java.util.Optional.ofNullable(decide.move(key))) + .orElse(null); String movedAttachments = attachmentsJson; boolean changed = false; @@ -197,11 +206,43 @@ private static String adoptReference( } } + /** Streams both files; a restore has no attachment size ceiling to load them whole under. */ private static boolean sameContent(@NonNull File first, @NonNull File second) throws IOException { - return first.length() == second.length() - && java.util.Arrays.equals( - Files.readAllBytes(first.toPath()), Files.readAllBytes(second.toPath())); + if (first.length() != second.length()) { + return false; + } + try (java.io.InputStream left = + new java.io.BufferedInputStream(new java.io.FileInputStream(first)); + java.io.InputStream right = + new java.io.BufferedInputStream(new java.io.FileInputStream(second))) { + byte[] leftBuffer = new byte[8192]; + byte[] rightBuffer = new byte[8192]; + while (true) { + int leftRead = readFully(left, leftBuffer); + int rightRead = readFully(right, rightBuffer); + if (leftRead != rightRead + || !java.util.Arrays.equals( + java.util.Arrays.copyOf(leftBuffer, leftRead), + java.util.Arrays.copyOf(rightBuffer, rightRead))) { + return false; + } + if (leftRead < leftBuffer.length) { + return true; + } + } + } + } + + private static int readFully(@NonNull java.io.InputStream input, @NonNull byte[] buffer) + throws IOException { + int filled = 0; + while (filled < buffer.length) { + int read = input.read(buffer, filled, buffer.length - filled); + if (read == -1) break; + filled += read; + } + return filled; } /** {@code name.ext} becomes {@code name-.ext}, still a safe single segment. */ diff --git a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java index bb5f5b5a..a09ebbdd 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java +++ b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java @@ -145,7 +145,9 @@ public void resolveConflict( @Override public SyncState sync(@NonNull String accessToken) { return new SyncService(store) - .sync(new GoogleDriveSyncBackend(accessToken)); + .sync( + new GoogleDriveSyncBackend(accessToken), + preferenceHelper::isSyncEnabled); } @Override diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 15891ab6..00897409 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -517,7 +517,18 @@ private void applyReceivedPreferences() { binding.getRoot() .postDelayed( () -> { - if (!isFinishing() && !isDestroyed() && !syncRunning) { + if (isFinishing() || isDestroyed() || syncRunning) { + return; + } + // The stored mode is pushed to AppCompat only here, once the sync + // result is in hand; pushing it from inside the apply recreated this + // screen mid-sync and dropped the result. A changed mode recreates + // the screen by itself; otherwise the rebuild is done here. + int before = + androidx.appcompat.app.AppCompatDelegate.getDefaultNightMode(); + themePreferencesCache.applyCurrentThemeMode(); + if (androidx.appcompat.app.AppCompatDelegate.getDefaultNightMode() + == before) { recreate(); } }, diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java index 973b428f..2005ea08 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java @@ -30,6 +30,15 @@ public class ZipBackupHelper { private static final String TAG = "ZipBackupHelper"; + /** + * The most an archive may unpack to, all entries together. + * + *

An archive is untrusted input, and ZIP compresses a run of zeros a thousandfold; without a + * ceiling a small file could fill the device. Generous for a real library, far below what a + * bomb wants. + */ + static final long MAX_TOTAL_UNCOMPRESSED_BYTES = 4L * 1024L * 1024L * 1024L; + /** The only entry shape an archive may place a file under: one note folder, one file name. */ private static final Pattern ATTACHMENT_ENTRY = Pattern.compile(Pattern.quote(ATTACHMENTS_BASE_DIR) + "/(note_[1-9][0-9]*)/([^/]+)"); @@ -113,7 +122,15 @@ public static JsonBackup readZipBackup(Context ctx, Uri uri) throws Exception { @NonNull static JsonBackup readZipBackup(@NonNull ZipInputStream zis, @NonNull File stagingRoot) throws IOException { + return readZipBackup(zis, stagingRoot, MAX_TOTAL_UNCOMPRESSED_BYTES); + } + + @NonNull + static JsonBackup readZipBackup( + @NonNull ZipInputStream zis, @NonNull File stagingRoot, long maxTotalBytes) + throws IOException { JsonBackup backup = null; + long[] total = {0L}; ZipEntry entry; while ((entry = zis.getNextEntry()) != null) { if (entry.getName().equals(FILE_NAME_BACKUP)) { @@ -121,6 +138,9 @@ static JsonBackup readZipBackup(@NonNull ZipInputStream zis, @NonNull File stagi byte[] tmp = new byte[4096]; int n; while ((n = zis.read(tmp)) != -1) { + if (!account(total, n, maxTotalBytes)) { + return new JsonBackup().error(); + } buffer.write(tmp, 0, n); } try { @@ -150,6 +170,10 @@ static JsonBackup readZipBackup(@NonNull ZipInputStream zis, @NonNull File stagi byte[] data = new byte[4096]; int n; while ((n = zis.read(data)) != -1) { + if (!account(total, n, maxTotalBytes)) { + Log.w(TAG, "The backup unpacks to more than the restore allows"); + return new JsonBackup().error(); + } fos.write(data, 0, n); } } @@ -159,6 +183,12 @@ static JsonBackup readZipBackup(@NonNull ZipInputStream zis, @NonNull File stagi return backup != null ? backup : new JsonBackup().error(); } + /** Adds to the running total; false once the ceiling is passed. */ + private static boolean account(long[] total, int read, long maxTotalBytes) { + total[0] += read; + return total[0] <= maxTotalBytes; + } + /** * Resolves one archive entry inside the staging directory, or {@code null} if it does not name * exactly one file in one note folder. diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 5ae9bbd4..6870fe45 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -707,7 +707,7 @@ public void publish_namesTheFrontierOfTheReadItQuotesAsParents() throws Exceptio @Test public void publish_retiresTheBundlesTheNewOneSupersedes() throws Exception { // Nothing ever deleted a bundle, so every sync downloaded and decoded the whole history - // to find one or two heads. Seeded bundles carry no publication time, which counts as old. + // to find one or two heads. Age is Drive's creation time, not a device's clock. SyncBundleCodec codec = new SyncBundleCodec(); byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); @@ -723,6 +723,7 @@ public void publish_retiresTheBundlesTheNewOneSupersedes() throws Exception { server.seedOwnedBundleBytes(base); server.seedOwnedBundleBytes(first); server.seedOwnedBundleBytes(second); + server.ageBundles(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); publish(backend(), snapshotWithTitle("Third")); @@ -765,6 +766,47 @@ public void publish_keepsASupersededBundleUntilTheGracePeriodHasPassed() throws assertThat(server.deletedFileIds()).hasSize(1); } + @Test + public void publish_neverPrunesABundleWhoseAgeDriveDoesNotReport() throws Exception { + // A device's own publication stamp used to stand in for the age, and a missing stamp + // counted as old — so a bundle another device was still fetching could be deleted under + // it. Only Drive's creation time is trusted now, and its absence means "keep". + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] head = + codec.encode( + snapshotWithTitle("Head"), CLOCK.instant(), Collections.singleton(baseId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(head); + server.ageBundles(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); + server.withholdCreatedTime(); + + publish(backend(), snapshotWithTitle("Next")); + + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.bundleCount()).isEqualTo(3); + } + + @Test + public void readAttachment_fallsBackToTheGoodCopyInAnotherRoot() throws Exception { + // Duplicate roots are a supported state. With Drive's checksums withheld the bytes must + // be read to tell the copies apart; handing over the first root's only candidate unread + // let a corrupt copy there shadow the good one in the other root on every sync. + byte[] good = "good bytes".getBytes(StandardCharsets.UTF_8); + String hash = sha256(good); + server.seedCorruptAttachment(hash, "corrupt".getBytes(StandardCharsets.UTF_8)); + server.registerAttachment(good); + server.seedOwnedBundle(snapshot(NOTE_ID, hash)); + server.withholdChecksums(); + assertThat(server.ownedFolderCount()).isEqualTo(2); + + try (java.io.InputStream restored = backend().readAttachment(hash)) { + assertThat(restored).isNotNull(); + assertThat(readAll(restored)).isEqualTo(good); + } + } + // ------------------------------------------------- attachment transfer cost @Test @@ -1165,6 +1207,7 @@ private static final class FakeDriveServer implements AutoCloseable { private final List deletedFileIds = java.util.Collections.synchronizedList(new ArrayList<>()); private volatile boolean withholdChecksums; + private volatile boolean withholdCreatedTime; private SyncSnapshot updateBeforeNextPatch; private final AtomicInteger nextId = new AtomicInteger(1); @@ -1251,11 +1294,11 @@ List deletedFileIds() { return new ArrayList<>(deletedFileIds); } - /** Marks every stored bundle as published {@code millis} ago relative to {@code now}. */ - void ageBundles(long now, long millis) { + /** Moves every stored bundle's Drive-side creation time {@code millis} into the past. */ + void ageBundles(long millis) { for (DriveFile file : files.values()) { if ("1".equals(file.appProperties.get("mynotesBundle"))) { - file.appProperties.put("mynotesBundlePublishedAt", Long.toString(now - millis)); + file.createdAtMillis -= millis; } } } @@ -1372,6 +1415,11 @@ void withholdChecksums() { withholdChecksums = true; } + /** Models a listing that reports no creation time for its files. */ + void withholdCreatedTime() { + withholdCreatedTime = true; + } + String registerAttachment(byte[] bytes) throws Exception { String hash = sha256(bytes); seededAttachmentContent.put(hash, bytes); @@ -1524,6 +1572,12 @@ && ownedFolderCount() == 0) { appProperties.addProperty(entry.getKey(), entry.getValue()); } value.add("appProperties", appProperties); + if (!withholdCreatedTime) { + // Drive's own clock, RFC 3339, as the real listing reports it. + value.addProperty( + "createdTime", + Instant.ofEpochMilli(file.createdAtMillis).toString()); + } array.add(value); } } @@ -1924,6 +1978,10 @@ public void close() { private static final class DriveFile { private final String id; + + /** Drive's creation time; the fake's server clock is the tests' fixed CLOCK. */ + private long createdAtMillis = CLOCK.millis(); + private byte[] content = new byte[0]; private String name; private String mimeType; diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java new file mode 100644 index 00000000..ccc8cc42 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java @@ -0,0 +1,136 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks; +import java.time.Instant; +import org.junit.Test; + +/** + * A note published by 2.6.50 has to come out of the decoder in the shape the upgraded store builds + * for the same, unchanged note. Otherwise the two hash differently at the same timestamp and the + * merge reports a conflict against the note itself — on every sync, when the old version wins. + */ +public class LegacyNotePayloadTest { + + private static final String NOTE_ID = "550e8400-e29b-41d4-a716-446655440000"; + private static final String HASH = + "d6f1f3d5d8cf9b5a4a2469787998dc45eb59f401b93b1b4cde4998dc409ebdc8"; + private static final String LOCAL_URL = "editorjs://attachments/note_7/1700000000000_1.png"; + + @Test + public void upgradesA2650PayloadToTheCurrentDerivationAndWireForm() { + JsonObject legacy = legacyPayload(LOCAL_URL, "photo.png"); + String expectedId = AttachmentLogicalIds.derive(NOTE_ID, 0, LOCAL_URL, "photo.png", HASH); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, legacy)).isTrue(); + + JsonObject entry = legacy.getAsJsonArray("attachmentsManifest").get(0).getAsJsonObject(); + assertThat(entry.get("id").getAsString()).isEqualTo(expectedId); + assertThat(legacy.getAsJsonObject("attachmentNames").get(expectedId).getAsString()) + .isEqualTo("photo.png"); + assertThat(EditorAttachmentBlocks.fileUrls(legacy.get("f").getAsString())) + .containsExactly(AttachmentWireUrl.forLogicalId(expectedId)); + assertThat(legacy.getAsJsonArray("attachmentHashes").get(0).getAsString()).isEqualTo(HASH); + } + + @Test + public void theUpgradedPayloadHashesLikeTheCurrentBuildOfTheSameNote() { + // What RoomSyncStore builds today for the same unchanged note: the same derivation over + // the same inputs, the block already in wire form, names keyed by the new id. + String id = AttachmentLogicalIds.derive(NOTE_ID, 0, LOCAL_URL, "photo.png", HASH); + JsonObject current = legacyPayload(LOCAL_URL, "photo.png"); + current.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("id", id); + JsonObject names = new JsonObject(); + names.addProperty(id, "photo.png"); + current.add("attachmentNames", names); + current.addProperty( + "f", + EditorAttachmentBlocks.rewriteUrls( + current.get("f").getAsString(), url -> AttachmentWireUrl.forLogicalId(id))); + JsonObject legacy = legacyPayload(LOCAL_URL, "photo.png"); + + LegacyNotePayload.upgrade(NOTE_ID, legacy); + + Instant at = Instant.parse("2026-08-31T12:00:01Z"); + assertThat( + SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, at, legacy) + .getCanonicalPayloadHash()) + .isEqualTo( + SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, at, current) + .getCanonicalPayloadHash()); + } + + @Test + public void leavesAPayloadWhoseIdsWereNotDerivedTheOldWayAlone() { + // An id a receiving device restored, or one already derived with the hash: not provably + // the old shape, so it is not touched. + JsonObject payload = legacyPayload(LOCAL_URL, "photo.png"); + payload.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("id", "7d444840-9dc0-11d1-b245-5ffdce74fad2"); + String before = payload.toString(); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, payload)).isFalse(); + assertThat(payload.toString()).isEqualTo(before); + } + + @Test + public void leavesAPayloadWhoseBlocksDoNotLineUpWithItsManifestAlone() { + JsonObject payload = legacyPayload(LOCAL_URL, "photo.png"); + payload.addProperty( + "f", + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + LOCAL_URL + + "\"}}},{\"type\":\"image\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_7/other.png\"}}}]"); + String before = payload.toString(); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, payload)).isFalse(); + assertThat(payload.toString()).isEqualTo(before); + } + + @Test + public void leavesAPayloadAlreadyInWireFormAlone() { + JsonObject payload = legacyPayload(AttachmentWireUrl.forLogicalId(NOTE_ID), "photo.png"); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, payload)).isFalse(); + } + + /** Exactly what 2.6.50's store built: local block URL, hash-less id, names keyed by it. */ + private static JsonObject legacyPayload(String blockUrl, String displayName) { + String legacyId = AttachmentLogicalIds.deriveLegacy(NOTE_ID, 0, blockUrl, displayName); + JsonObject payload = new JsonObject(); + payload.addProperty("b", "Shopping"); + payload.addProperty("c", "Milk"); + payload.addProperty( + "f", + "[{\"id\":\"blk\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + blockUrl + + "\",\"name\":\"" + + displayName + + "\"}}}]"); + JsonObject entry = new JsonObject(); + entry.addProperty("id", legacyId); + entry.addProperty("sha256", HASH); + entry.addProperty("mimeType", "image/png"); + entry.addProperty("size", 42L); + entry.addProperty("path", "attachments/" + HASH); + entry.addProperty("displayName", displayName); + JsonArray manifest = new JsonArray(); + manifest.add(entry); + payload.add("attachmentsManifest", manifest); + JsonArray hashes = new JsonArray(); + hashes.add(HASH); + payload.add("attachmentHashes", hashes); + JsonObject names = new JsonObject(); + names.addProperty(legacyId, displayName); + payload.add("attachmentNames", names); + return payload; + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java index 10d15f81..22de4a23 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java @@ -491,6 +491,100 @@ public T run( eq("[moved]")); } + @Test + public void insertNotes_adoptsStagedFilesForANoteThisDeviceAlreadyHas() { + // A row can exist with its files gone — restored from a JSON backup, or a cleared + // attachment folder. The archive is the only place the files are, and skipping the note + // as already present used to skip its files too; the staged copies were then thrown away + // at the next restore, after a "restore OK". + List relocatedFrom = new ArrayList<>(); + SyncMutationCoordinator staging = + new SyncMutationCoordinator( + new SyncMutationCoordinator.TransactionExecutor() { + @Override + public T run( + SyncMutationCoordinator.TransactionCallable callable) { + return callable.call(); + } + }, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + new FixedTimeProvider(1_000L), + new QueueStableIdGenerator("stable-a"), + (note, previousId) -> { + relocatedFrom.add(previousId); + return false; + }); + Note existing = new Note().create("Title", "Body", 10L, "work"); + existing.setId(5); + Note fromBackup = new Note().create("Title", "Body", 10L, "work"); + fromBackup.setId(5); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); + + staging.insertNotes(new ArrayList<>(List.of(fromBackup))); + + verify(noteDao, never()).addNotes(anyList()); + assertThat(relocatedFrom).containsExactly(5); + // Nothing was rewritten, so the row is left exactly as it was. + verify(noteDao, never()) + .updateNoteContent( + org.mockito.ArgumentMatchers.anyInt(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.anyLong(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any()); + } + + @Test + public void insertNotes_storesAPresentNoteAgainWhenAdoptionRewroteAReference() { + SyncMutationCoordinator staging = + new SyncMutationCoordinator( + new SyncMutationCoordinator.TransactionExecutor() { + @Override + public T run( + SyncMutationCoordinator.TransactionCallable callable) { + return callable.call(); + } + }, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + new FixedTimeProvider(1_000L), + new QueueStableIdGenerator("stable-a"), + (note, previousId) -> { + note.setAttachments("[renamed]"); + return true; + }); + Note existing = new Note().create("Title", "Body", 10L, "work"); + existing.setId(5); + Note fromBackup = new Note().create("Title", "Body", 10L, "work"); + fromBackup.setId(5); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); + + staging.insertNotes(new ArrayList<>(List.of(fromBackup))); + + verify(noteDao) + .updateNoteContent( + eq(5), + eq("Title"), + eq("Body"), + org.mockito.ArgumentMatchers.any(), + eq(10L), + eq("work"), + eq("[renamed]")); + // The row changed, so sync has to publish it. + assertThat(syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 5L)).isNotNull(); + } + private static final class FixedTimeProvider implements SyncMutationCoordinator.TimeProvider { private final long value; diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index 5ece616e..a406d0b3 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -361,6 +361,51 @@ public void sync_pinsConflictBlobsWithoutTransferringOrVerifyingThemAgain() thro assertThat(java.util.Collections.frequency(store.events, "writeAttachment")).isEqualTo(1); } + @Test + public void sync_refusesToStartOnceSyncWasTurnedOffEvenWithTheLockInHand() { + // Disconnect turns sync off and wipes the account's state under the lock; a worker that + // took the lock afterwards wrote the old account's state back. Nothing may be persisted. + FakeStore store = new FakeStore(snapshot(note(TEN, "Local"))); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState state = + new SyncService(store, new SyncMerger(), CLOCK).sync(backend, () -> false); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(state.getErrorMessage()).contains("turned off"); + assertThat(store.states).isEmpty(); + assertThat(store.applyCalls).isEqualTo(0); + assertThat(backend.events).isEmpty(); + } + + @Test + public void sync_namesTheNoteWhenABlobNoEndpointHoldsWasPublishedFromMemory() throws Exception { + // The store described the attachment from the hash and size its column remembered, + // expecting the bytes back from Drive. When Drive has nothing either — a different + // account, say — the failure has to name the note, as the build failure it replaced did, + // not leave the user with a hash and a sync that fails forever. + byte[] bytes = "gone everywhere".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + FakeStore store = + new FakeStore(snapshot(noteWithAttachment(TEN, "Body", hash, bytes.length))); + store.attachmentHashes = Collections.singletonList(hash); + store.rememberedOnly = + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + "Shopping list"); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(state.getErrorMessage()) + .isEqualTo( + "Local snapshot is incomplete: MISSING_ATTACHMENT in note \"Shopping list\""); + assertThat(backend.writeSnapshotCalls).isEqualTo(0); + } + private static SyncSnapshot snapshot(SyncRecord... records) { return new SyncSnapshot(Arrays.asList(records)); } @@ -526,6 +571,14 @@ public boolean hasDurableAttachment(String sha256, long sizeBytes) { return durable.contains(sha256); } + /** Set when the store described an attachment from remembered metadata only. */ + private SnapshotProblem rememberedOnly; + + @Override + public SnapshotProblem describeMissingAttachment(String sha256) { + return rememberedOnly; + } + @Override public SyncState readState() { if (readStateFailure != null) { diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java index 7d84f0f4..3acd4de6 100644 --- a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java @@ -122,6 +122,35 @@ public void neverOverwritesAFileAnotherNoteAlreadyShows() throws Exception { assertThat(result.attachmentsJson).contains("note_7/" + adoptedName); } + @Test + public void columnAndBlocksAgreeOnTheAdoptedNameWhenAFileCollides() throws Exception { + // The mover runs once for the column and once for the same URL in the blocks. Picking a + // fresh name on each call made two copies and pointed column and blocks at different + // files; the cleaner then deleted the one the column did not list. + seed(7, "photo.jpg", "mine"); + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 7, "photo.jpg", "foreign"); + String blocks = + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + AttachmentStorage.urlFor(7, "photo.jpg") + + "\",\"name\":\"photo.jpg\"}}}]"; + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 7, 7, attachmentsJson(7, "photo.jpg"), blocks); + + String columnUrl = + com.google.gson.JsonParser.parseString(result.attachmentsJson) + .getAsJsonArray() + .get(0) + .getAsJsonObject() + .get("url") + .getAsString(); + assertThat(EditorAttachmentBlocks.fileUrls(result.valueJson)).containsExactly(columnUrl); + // Exactly two files: the one that was there and the one adopted, no second copy. + assertThat(new File(root, "note_7").listFiles()).hasLength(2); + } + @Test public void reusesAnIdenticalFileInsteadOfDuplicatingIt() throws Exception { seed(7, "photo.jpg", "same bytes"); diff --git a/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java b/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java index fb4d2061..24c7e3e7 100644 --- a/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java +++ b/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java @@ -89,6 +89,22 @@ public void anArchiveWithoutTheJsonIsAnError() throws Exception { assertThat(ZipBackupHelper.readZipBackup(zip(entries), staging).isError()).isTrue(); } + @Test + public void refusesAnArchiveThatUnpacksPastTheCeiling() throws Exception { + // ZIP compresses a run of zeros a thousandfold; an archive is untrusted input and a + // small one could fill the device. + File staging = new File(temporaryFolder.getRoot(), "staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put( + Backup.FILE_NAME_BACKUP, + new Gson().toJson(new JsonBackup()).getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_5/zeros.bin", new byte[64 * 1024]); + + JsonBackup backup = ZipBackupHelper.readZipBackup(zip(entries), staging, 16 * 1024); + + assertThat(backup.isError()).isTrue(); + } + private static ZipInputStream zip(Map entries) throws Exception { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ZipOutputStream zip = new ZipOutputStream(bytes)) { From fb56963a8f3444bc2766096e529f032790f7c104 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Sat, 5 Sep 2026 10:57:54 +0300 Subject: [PATCH 3/4] fix(sync): keep both blobs, time pruning from supersession, accept our own backups Third review of the fix rounds, three findings plus one from the device. Aliasing a colliding attachment id had replaced an encode-time refusal with silent loss: when one note carried two manifest entries under the same logical id with different content, both were re-keyed to the second entry's alias and every receiver wrote the second blob's bytes for both references. Wire ids are now recorded per manifest position, so both blobs travel; a note that references one manifest entry twice is refused by the validator, which is the safe failure the old code had; and the store derives a fresh id rather than emitting a repeated one. Bundle pruning measured the grace from when a bundle was created, not from when it was superseded, so one created days ago and superseded seconds ago was deleted while another device was mid-read. Bundles outside the frontier are now marked on Drive when a read finds them, and only a bundle already marked when this sync read it, whose mark is older than the grace, is deleted. A blob whose Drive checksum matched but whose listing size was missing was treated as corrupt instead of being read and verified, turning a good file into a permanent sync failure and a duplicate upload every sync. Restoring a backup was refused when the name did not end in .mnbkn, and Android's SAF appends " (2)" to a second backup saved in the same folder, so the app rejected its own file. The name is now a fast accept and the decision falls back to opening the document and looking for the backup JSON inside the archive. 325 unit tests, 80 instrumentation tests, 0 failures; lint 0 errors. --- .../data/sync/GoogleDriveSyncBackend.java | 95 +++++++-- .../mynotes/data/sync/RoomSyncStore.java | 6 +- .../mynotes/data/sync/SyncBundleCodec.java | 28 +-- .../data/sync/SyncBundleValidator.java | 7 + .../backup/local/BackupFileValidator.java | 72 ++++++- .../data/sync/GoogleDriveSyncBackendTest.java | 188 +++++++++++------- .../data/sync/SyncBundleCodecTest.java | 82 ++++++++ .../backup/local/BackupFileValidatorTest.java | 79 ++++++++ 8 files changed, 443 insertions(+), 114 deletions(-) create mode 100644 app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index dbf888ec..838719b4 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -45,6 +45,14 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final String PROPERTY_BUNDLE = "mynotesBundle"; private static final String PROPERTY_BUNDLE_PUBLISHED_AT = "mynotesBundlePublishedAt"; private static final String PROPERTY_ATTACHMENT_SHA256 = "mynotesAttachmentSha256"; + + /** + * Set on a bundle the first time a read finds it outside the frontier. Drive stamps the update + * with its own {@code modifiedTime}, which is therefore the moment the bundle was seen to be + * superseded — measured by Drive's clock, not by whichever device happened to publish. + */ + private static final String PROPERTY_BUNDLE_SUPERSEDED = "mynotesBundleSuperseded"; + private static final int MAX_BUNDLE_RESPONSE_BYTES = 32 * 1024 * 1024; private static final long MAX_ATTACHMENT_RESPONSE_BYTES = SyncBundleValidator.MAX_ATTACHMENT_BYTES; @@ -55,11 +63,13 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final int MAX_ERROR_DETAIL_CHARS = 200; /** - * How long a superseded bundle stays after its successor appears. + * How long a superseded bundle stays after it was first seen to be superseded. * *

A device that listed the folder just before the successor was published may still be * fetching the old bundle; an hour outlives any read, including the six-hourly worker's, which - * WorkManager stops after ten minutes. + * WorkManager stops after ten minutes. Measured from the supersession mark, not from the + * bundle's creation: a head created days ago and superseded seconds ago is exactly the file + * another device is most likely to be reading right now. */ static final long BUNDLE_PRUNE_GRACE_MILLIS = 60L * 60L * 1000L; @@ -239,12 +249,48 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { SyncMergeResult.Source.REMOTE)); } + markSupersededBundles(bundleFiles, frontier); lastReadBundles = Collections.unmodifiableList(bundleFiles); lastReadToken = UUID.randomUUID().toString(); return new RemoteSnapshot( merged, conflicts, frontier, alternatives, resolvedAlternativeIds, lastReadToken); } + /** + * Stamps every bundle that has left the frontier, once, so its grace period starts now. + * + *

Best effort: a bundle that cannot be marked is never pruned, which costs a download per + * sync and nothing else. + */ + private void markSupersededBundles( + @NonNull List bundles, @NonNull Collection frontier) { + for (BundleFile bundle : bundles) { + if (frontier.contains(bundle.logicalId) || bundle.supersededAtMillis != null) { + continue; + } + try { + JsonObject patch = new JsonObject(); + patch.add("appProperties", appProperties(PROPERTY_BUNDLE_SUPERSEDED, "1")); + // HttpURLConnection has no PATCH; Google's APIs honour the override header. + HttpURLConnection connection = + open("POST", apiBase + "/files/" + bundle.fileId + "?fields=id"); + connection.setRequestProperty("X-HTTP-Method-Override", "PATCH"); + connection.setRequestProperty("Content-Type", MIME_JSON); + connection.setDoOutput(true); + try { + try (OutputStream output = connection.getOutputStream()) { + output.write(jsonBytes(patch)); + } + ensureSuccess(connection); + } finally { + connection.disconnect(); + } + } catch (IOException ignored) { + // Marked at the next read instead; the grace period simply starts later. + } + } + } + @Override public synchronized void publish(@NonNull SyncPublication publication) throws IOException { // Causal parents used to come from a mutable field, so a write with no preceding read @@ -297,9 +343,9 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO * sync, and every later sync downloaded, unzipped and validated all of them to compute a * frontier of one or two heads. A bundle is a complete snapshot, so everything a superseded * bundle held — records, tombstones, unresolved alternatives — lives on in its descendants, and - * the read path already tolerates a missing ancestor. The heads this publish descended from are - * kept for now: they are what a concurrent publisher is about to name as parents. They go at - * the next sync, once the grace period has passed. + * the read path already tolerates a missing ancestor. A bundle goes only once it has been + * marked superseded for the whole grace period; the heads this publish descended from are not + * even marked yet, and are what a concurrent publisher is about to name as parents. * *

Best effort by design: a bundle that cannot be removed costs a download next time, never * correctness. @@ -307,12 +353,12 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO private void pruneSupersededBundles(@NonNull Collection frontierBundleIds) { long cutoff = clock.millis() - BUNDLE_PRUNE_GRACE_MILLIS; for (BundleFile bundle : lastReadBundles) { - // The age comes from Drive's own creation time, never from a property the publishing - // device stamped with its clock; a bundle whose age Drive does not report is left - // alone rather than assumed old. + // Only a bundle that was already marked superseded when this sync read it, with the + // mark's Drive-side timestamp older than the grace, may go. A bundle marked during + // this very read, or one whose mark Drive does not date, stays. if (frontierBundleIds.contains(bundle.logicalId) - || bundle.createdAtMillis == null - || bundle.createdAtMillis > cutoff) { + || bundle.supersededAtMillis == null + || bundle.supersededAtMillis > cutoff) { continue; } try { @@ -637,20 +683,22 @@ private static List computeFrontier( private static final class BundleFile { private final String fileId; @Nullable private final String logicalId; - @Nullable private final Long createdAtMillis; + + /** Drive's modifiedTime of the supersession mark, or null when unmarked or undated. */ + @Nullable private final Long supersededAtMillis; private BundleFile( @NonNull String fileId, @Nullable String logicalId, - @Nullable Long createdAtMillis) { + @Nullable Long supersededAtMillis) { this.fileId = fileId; this.logicalId = logicalId; - this.createdAtMillis = createdAtMillis; + this.supersededAtMillis = supersededAtMillis; } @NonNull private BundleFile withLogicalId(@NonNull String id) { - return new BundleFile(fileId, id, createdAtMillis); + return new BundleFile(fileId, id, supersededAtMillis); } } @@ -659,28 +707,30 @@ private List findBundles(@NonNull String folderId) throws IOExceptio JsonArray bundles = listFiles( ownedFilesQuery(folderId, PROPERTY_BUNDLE, "1"), - "files(id,name,createdTime)"); + "files(id,name,modifiedTime,appProperties)"); List result = new ArrayList<>(bundles.size()); for (int index = 0; index < bundles.size(); index++) { JsonObject file = bundles.get(index).getAsJsonObject(); + JsonObject properties = file.getAsJsonObject("appProperties"); + boolean marked = properties != null && properties.has(PROPERTY_BUNDLE_SUPERSEDED); result.add( new BundleFile( file.get("id").getAsString(), null, - createdAtOf(optionalString(file, "createdTime")))); + marked ? instantOf(optionalString(file, "modifiedTime")) : null)); } result.sort(Comparator.comparing(file -> file.fileId)); return result; } - /** Drive reports {@code createdTime} as RFC 3339; anything else counts as unknown. */ + /** Drive reports times as RFC 3339; anything else counts as unknown. */ @Nullable - private static Long createdAtOf(@Nullable String createdTime) { - if (createdTime == null) { + private static Long instantOf(@Nullable String time) { + if (time == null) { return null; } try { - return java.time.Instant.parse(createdTime).toEpochMilli(); + return java.time.Instant.parse(time).toEpochMilli(); } catch (RuntimeException malformed) { return null; } @@ -741,8 +791,11 @@ private String findVerifiedAttachment( if (isVerifiedWithoutReading(candidate, sha256, expectedSize)) { return candidate.id; } - if (candidate.sha256Checksum != null) { + if (candidate.sha256Checksum != null && !candidate.sha256Checksum.equals(sha256)) { // Drive's own digest disagrees with the index; reading would only confirm it. + // Anything less definite — a matching digest without a usable size, or one the + // expected size disagrees with — is read and verified like an unlisted object, + // rather than counted as corrupt and re-uploaded on every sync. continue; } try (InputStream content = openAttachment(candidate.id)) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 4885eadd..c5a95150 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -1511,6 +1511,7 @@ private boolean addAttachmentMetadata( JsonArray hashes = new JsonArray(); JsonObject names = new JsonObject(); Map logicalIdByUrl = new HashMap<>(); + Set logicalIdsInNote = new HashSet<>(); boolean complete = true; for (int attachmentIndex = 0; attachmentIndex < attachments.size(); attachmentIndex++) { JsonElement element = attachments.get(attachmentIndex); @@ -1600,7 +1601,9 @@ private boolean addAttachmentMetadata( } String displayName = displayNameFor(attachment, file, hash); String logicalId = attachment.id; - if (!isCanonicalUuid(logicalId)) { + // A column entry may repeat an id — a duplicated block whose file was later replaced. + // The manifest is keyed by id, so the repeat is given its own, as if it had none. + if (!isCanonicalUuid(logicalId) || logicalIdsInNote.contains(logicalId)) { // Existing editor data predates logical attachment IDs. Deriving from the stable // note, source URL, position and content keeps the migration deterministic while // allowing equal-content references to remain distinct logical attachments. The @@ -1620,6 +1623,7 @@ private boolean addAttachmentMetadata( displayName, hash); } + logicalIdsInNote.add(logicalId); hashes.add(hash); names.addProperty(logicalId, displayName); logicalIdByUrl.put(comparableUrl(attachment.url), logicalId); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 285aea71..8419981d 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -226,20 +226,24 @@ private static JsonArray liveArray( /** * Replaces the local attachment fields with the wire references. * - * @param wireIds the manifest id each of this record's logical attachment ids travels under; - * identical to the logical id except for a re-keyed collision. + * @param wireIds the manifest id each of this record's attachments travels under, by manifest + * position; identical to the logical id except for a re-keyed collision. */ private static void normalizeNoteAttachmentFields( - JsonObject note, @NonNull Map wireIds) throws IOException { + JsonObject note, @NonNull List wireIds) throws IOException { JsonArray manifestEntries = note.getAsJsonArray("attachmentsManifest"); JsonArray attachmentIds = new JsonArray(); JsonObject attachmentNames = new JsonObject(); JsonObject aliases = new JsonObject(); if (manifestEntries != null) { - for (JsonElement element : manifestEntries) { + for (int index = 0; index < manifestEntries.size(); index++) { AttachmentManifestEntry attachment = - AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - String wireId = wireIds.getOrDefault(attachment.id, attachment.id); + AttachmentManifestEntry.fromJson( + manifestEntries.get(index).getAsJsonObject()); + // By position, not by logical id: one record can carry the same id twice with + // different content, and a map keyed by id sent both references to the second + // blob, so every receiver lost the first one's bytes. + String wireId = index < wireIds.size() ? wireIds.get(index) : attachment.id; attachmentIds.add(wireId); if (!wireId.equals(attachment.id)) { aliases.addProperty(wireId, attachment.id); @@ -365,11 +369,11 @@ private static AttachmentPlan planAttachments( throw new IOException( "Two notes reference conflicting attachment metadata"); } - plan.wireIdsFor(record).put(attachment.id, wireId); } if (sameId == null) { plan.byWireId.put(wireId, attachment.withId(wireId)); } + plan.wireIdsFor(record).add(wireId); AttachmentManifestEntry previous = seenByHash.putIfAbsent(attachment.sha256, attachment); if (previous != null && !previous.sameRemoteFile(attachment)) { @@ -392,17 +396,17 @@ private static String aliasFor(@NonNull AttachmentManifestEntry attachment) { .toString(); } - /** The manifest entries by wire id, and each record's logical-to-wire id mapping. */ + /** The manifest entries by wire id, and each record's wire ids in manifest order. */ private static final class AttachmentPlan { private final Map byWireId = new LinkedHashMap<>(); - private final Map> wireIdsByRecord = + private final Map> wireIdsByRecord = new java.util.IdentityHashMap<>(); @NonNull - Map wireIdsFor(@NonNull SyncRecord record) { - Map wireIds = wireIdsByRecord.get(record); + List wireIdsFor(@NonNull SyncRecord record) { + List wireIds = wireIdsByRecord.get(record); if (wireIds == null) { - wireIds = new LinkedHashMap<>(); + wireIds = new ArrayList<>(); wireIdsByRecord.put(record, wireIds); } return wireIds; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index 6df66d36..bdd36aad 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -176,11 +176,18 @@ private static void validateAttachmentReferences( } JsonObject attachmentNames = note.getAsJsonObject("attachmentNames"); validateAttachmentIdAliases(note, attachmentsById); + Set withinNote = new LinkedHashSet<>(); for (JsonElement element : attachmentIds) { if (element == null || !element.isJsonPrimitive()) { throw new IOException("Sync note attachmentIds entry is invalid"); } String attachmentId = element.getAsString(); + if (!withinNote.add(attachmentId)) { + // Two references to one manifest entry can only mean two attachments were + // collapsed into one on the way out; accepting it made every receiver write + // one blob's bytes for both. + throw new IOException("Sync note references one attachment twice"); + } SyncBundleCodec.AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); if (attachment == null) { throw new IOException("Note references an unknown attachment manifest entry"); diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java index 74e8d6bd..43bc6d72 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java @@ -5,11 +5,24 @@ import android.net.Uri; import android.provider.OpenableColumns; import android.util.Log; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.pasich.mynotes.R; +import com.pasich.mynotes.utils.constants.Backup; +import java.io.IOException; +import java.io.InputStream; +import java.util.Locale; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; /** - * Small helper class for validating selected backup files. Ensures that the file exists and has a - * supported extension. + * Decides whether a picked document is a backup this app can restore. + * + *

The decision used to rest on the display name's extension alone. Android's document picker + * appends " (2)" when a second backup is saved into a folder that already holds one, so the app + * refused its own freshly written file. The content decides now: an archive holding the backup JSON + * is a backup whatever it is called; the extension is only kept as the fast path for the legacy + * non-archive formats. */ public class BackupFileValidator { @@ -19,12 +32,18 @@ public class BackupFileValidator { private static final String EXT_ZIP = ".zip"; private static final String EXT_MNBK = ".mnbkn"; + /** Opens the picked document, so the content can be inspected. */ + public interface ContentOpener { + @Nullable + InputStream open() throws IOException; + } + /** - * Validate backup file based on its filename and extension. + * Validate a picked backup file. * *

- If user cancels selection → return silently (no errors shown). - If filename cannot be - * determined → callback.onInvalid(...) - If extension unsupported → callback.onInvalid(...) - - * If valid → callback.onValid(filename) + * determined → callback.onInvalid(...) - If neither name nor content is a backup → + * callback.onInvalid(...) - If valid → callback.onValid(filename) */ public static void isValidBackupFile(Context ctx, Uri uri, BackupValidatorCallback callback) { @@ -37,12 +56,7 @@ public static void isValidBackupFile(Context ctx, Uri uri, BackupValidatorCallba return; } - String lower = name.toLowerCase(); - - boolean ok = - lower.endsWith(EXT_JSON) || lower.endsWith(EXT_ZIP) || lower.endsWith(EXT_MNBK); - - if (!ok) { + if (!isAcceptable(name, () -> ctx.getContentResolver().openInputStream(uri))) { callback.onInvalid(ctx.getString(R.string.file_wrong_format)); return; } @@ -50,6 +64,42 @@ public static void isValidBackupFile(Context ctx, Uri uri, BackupValidatorCallba callback.onValid(name); } + /** + * The decision itself, free of {@code android.*} so every branch runs under a plain JVM test. + * + * @param name the document's display name, which the picker may have decorated. + * @param content the document's bytes, consulted when the name alone does not settle it. + */ + static boolean isAcceptable(@NonNull String name, @NonNull ContentOpener content) { + String lower = name.toLowerCase(Locale.ROOT); + if (lower.endsWith(EXT_JSON) || lower.endsWith(EXT_ZIP) || lower.endsWith(EXT_MNBK)) { + return true; + } + try (InputStream input = content.open()) { + return input != null && isBackupArchive(input); + } catch (IOException | RuntimeException unreadable) { + Log.w(TAG, "Could not inspect the picked document", unreadable); + return false; + } + } + + /** True when the bytes are a ZIP archive holding the backup JSON entry. */ + static boolean isBackupArchive(@NonNull InputStream input) throws IOException { + try (ZipInputStream zip = new ZipInputStream(input)) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (Backup.FILE_NAME_BACKUP.equals(entry.getName())) { + return true; + } + zip.closeEntry(); + } + } catch (RuntimeException notAnArchive) { + // A ZipInputStream over arbitrary bytes throws on a malformed entry header. + return false; + } + return false; + } + /** * Extract filename from content Uri via OpenableColumns. * diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 6870fe45..01b0ff99 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -705,9 +705,13 @@ public void publish_namesTheFrontierOfTheReadItQuotesAsParents() throws Exceptio } @Test - public void publish_retiresTheBundlesTheNewOneSupersedes() throws Exception { + public void publish_retiresABundleOnlyOnceItHasBeenSupersededForTheWholeGrace() + throws Exception { // Nothing ever deleted a bundle, so every sync downloaded and decoded the whole history - // to find one or two heads. Age is Drive's creation time, not a device's clock. + // to find one or two heads. A bundle is marked the first time a read finds it outside + // the frontier; Drive dates the mark, and the grace runs from there — not from the + // bundle's creation. A head created days ago and superseded seconds ago is exactly the + // file another device is most likely to be reading. SyncBundleCodec codec = new SyncBundleCodec(); byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); @@ -723,34 +727,18 @@ public void publish_retiresTheBundlesTheNewOneSupersedes() throws Exception { server.seedOwnedBundleBytes(base); server.seedOwnedBundleBytes(first); server.seedOwnedBundleBytes(second); - server.ageBundles(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); + // Created long ago; superseded only as far as this sync can tell. + server.ageBundles(3L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); publish(backend(), snapshotWithTitle("Third")); - // The head the publish descended from is kept: a concurrent publisher is about to name - // it as a parent. Its ancestors are gone. - assertThat(server.bundleCount()).isEqualTo(2); - assertThat(server.deletedFileIds()).hasSize(2); - assertThat( - backend() - .readSnapshot() - .find(SyncRecord.Type.NOTE, NOTE_ID) - .getPayload() - .get("title") - .getAsString()) - .isEqualTo("Third"); - } - - @Test - public void publish_keepsASupersededBundleUntilTheGracePeriodHasPassed() throws Exception { - GoogleDriveSyncBackend recent = backend(); - publish(recent, snapshotWithTitle("First")); - publish(recent, snapshotWithTitle("Second")); - - // Both were published by this clock moments ago; a device that listed the folder just - // before may still be reading the older one. - assertThat(server.bundleCount()).isEqualTo(2); + // Old by creation, but their supersession was only just recorded: nothing goes yet. + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.supersededBundleCount()).isEqualTo(2); + assertThat(server.bundleCount()).isEqualTo(4); + // Two hours on, by Drive's clock and this device's alike. + server.advanceClock(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); GoogleDriveSyncBackend later = new GoogleDriveSyncBackend( "token", @@ -758,19 +746,24 @@ public void publish_keepsASupersededBundleUntilTheGracePeriodHasPassed() throws server.uploadBase(), Clock.offset(CLOCK, java.time.Duration.ofHours(2)), new SyncBundleCodec()); - publish(later, snapshotWithTitle("Third")); + publish(later, snapshotWithTitle("Fourth")); - // Two hours on, the first is an ancestor nobody can still be fetching; the second is the - // head this publish descended from and stays for one more round. - assertThat(server.bundleCount()).isEqualTo(2); - assertThat(server.deletedFileIds()).hasSize(1); + // base and first were marked two hours ago and go; second was superseded by Third and + // is only marked now; Third is the head this publish descends from. + assertThat(server.deletedFileIds()).hasSize(2); + assertThat(server.bundleCount()).isEqualTo(3); + assertThat( + backend() + .readSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID) + .getPayload() + .get("title") + .getAsString()) + .isEqualTo("Fourth"); } @Test - public void publish_neverPrunesABundleWhoseAgeDriveDoesNotReport() throws Exception { - // A device's own publication stamp used to stand in for the age, and a missing stamp - // counted as old — so a bundle another device was still fetching could be deleted under - // it. Only Drive's creation time is trusted now, and its absence means "keep". + public void publish_neverPrunesABundleWhoseSupersessionDriveDoesNotDate() throws Exception { SyncBundleCodec codec = new SyncBundleCodec(); byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); @@ -779,32 +772,22 @@ public void publish_neverPrunesABundleWhoseAgeDriveDoesNotReport() throws Except snapshotWithTitle("Head"), CLOCK.instant(), Collections.singleton(baseId)); server.seedOwnedBundleBytes(base); server.seedOwnedBundleBytes(head); - server.ageBundles(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); - server.withholdCreatedTime(); - + server.withholdModifiedTime(); publish(backend(), snapshotWithTitle("Next")); + server.advanceClock(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); - assertThat(server.deletedFileIds()).isEmpty(); - assertThat(server.bundleCount()).isEqualTo(3); - } - - @Test - public void readAttachment_fallsBackToTheGoodCopyInAnotherRoot() throws Exception { - // Duplicate roots are a supported state. With Drive's checksums withheld the bytes must - // be read to tell the copies apart; handing over the first root's only candidate unread - // let a corrupt copy there shadow the good one in the other root on every sync. - byte[] good = "good bytes".getBytes(StandardCharsets.UTF_8); - String hash = sha256(good); - server.seedCorruptAttachment(hash, "corrupt".getBytes(StandardCharsets.UTF_8)); - server.registerAttachment(good); - server.seedOwnedBundle(snapshot(NOTE_ID, hash)); - server.withholdChecksums(); - assertThat(server.ownedFolderCount()).isEqualTo(2); + publish( + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + Clock.offset(CLOCK, java.time.Duration.ofHours(2)), + new SyncBundleCodec()), + snapshotWithTitle("Later")); - try (java.io.InputStream restored = backend().readAttachment(hash)) { - assertThat(restored).isNotNull(); - assertThat(readAll(restored)).isEqualTo(good); - } + // Marked, but Drive reports no time for the mark: nothing can be proven old enough. + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.bundleCount()).isEqualTo(4); } // ------------------------------------------------- attachment transfer cost @@ -844,6 +827,24 @@ public void hasVerifiedAttachment_trustsDrivesOwnChecksumWithoutDownloading() th assertThat(server.mediaReadsOfAttachment(claimed)).isEqualTo(0); } + @Test + public void hasVerifiedAttachment_readsTheBlobWhenTheListingCannotConfirmItsSize() + throws Exception { + // Drive's digest matches but the listing carries no usable size. Treating that as corrupt + // reported a good blob absent, so the service uploaded a duplicate on every sync and then + // failed anyway when the duplicate listed the same way. + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + server.withholdSizes(); + GoogleDriveSyncBackend backend = backend(); + + assertThat(backend.hasVerifiedAttachment(hash, (long) bytes.length)).isTrue(); + + assertThat(server.mediaReadsOfAttachment(hash)).isEqualTo(1); + assertThat(server.ownedAttachmentCount(hash)).isEqualTo(1); + } + @Test public void oneSyncListsTheRootFoldersOnce() throws Exception { byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); @@ -1207,7 +1208,12 @@ private static final class FakeDriveServer implements AutoCloseable { private final List deletedFileIds = java.util.Collections.synchronizedList(new ArrayList<>()); private volatile boolean withholdChecksums; - private volatile boolean withholdCreatedTime; + private volatile boolean withholdSizes; + private volatile boolean withholdModifiedTime; + + /** Drive's clock, as the fake stamps files with it; starts at the tests' fixed CLOCK. */ + private volatile long serverNowMillis = CLOCK.millis(); + private SyncSnapshot updateBeforeNextPatch; private final AtomicInteger nextId = new AtomicInteger(1); @@ -1294,11 +1300,12 @@ List deletedFileIds() { return new ArrayList<>(deletedFileIds); } - /** Moves every stored bundle's Drive-side creation time {@code millis} into the past. */ + /** Moves every stored bundle's Drive-side timestamps {@code millis} into the past. */ void ageBundles(long millis) { for (DriveFile file : files.values()) { if ("1".equals(file.appProperties.get("mynotesBundle"))) { file.createdAtMillis -= millis; + file.modifiedAtMillis -= millis; } } } @@ -1415,9 +1422,31 @@ void withholdChecksums() { withholdChecksums = true; } - /** Models a listing that reports no creation time for its files. */ - void withholdCreatedTime() { - withholdCreatedTime = true; + /** Models a listing that reports no modification time for its files. */ + void withholdModifiedTime() { + withholdModifiedTime = true; + } + + /** Models objects whose listing carries a checksum but no usable size. */ + void withholdSizes() { + withholdSizes = true; + } + + /** Moves Drive's clock forward; later stamps are dated from the new time. */ + void advanceClock(long millis) { + serverNowMillis += millis; + } + + /** Whether the client has marked the newest {@code count} bundles as superseded. */ + int supersededBundleCount() { + int count = 0; + for (DriveFile file : files.values()) { + if ("1".equals(file.appProperties.get("mynotesBundle")) + && file.appProperties.containsKey("mynotesBundleSuperseded")) { + count++; + } + } + return count; } String registerAttachment(byte[] bytes) throws Exception { @@ -1562,7 +1591,9 @@ && ownedFolderCount() == 0) { // stored bytes exactly as Drive does, so a corrupt object is exposed by its // real digest rather than by the property the uploader claimed. if (!withholdChecksums) { - value.addProperty("size", String.valueOf(file.content.length)); + if (!withholdSizes) { + value.addProperty("size", String.valueOf(file.content.length)); + } if (!"application/vnd.google-apps.folder".equals(file.mimeType)) { value.addProperty("sha256Checksum", sha256Unchecked(file.content)); } @@ -1572,11 +1603,13 @@ && ownedFolderCount() == 0) { appProperties.addProperty(entry.getKey(), entry.getValue()); } value.add("appProperties", appProperties); - if (!withholdCreatedTime) { - // Drive's own clock, RFC 3339, as the real listing reports it. + // Drive's own clock, RFC 3339, as the real listing reports it. + value.addProperty( + "createdTime", Instant.ofEpochMilli(file.createdAtMillis).toString()); + if (!withholdModifiedTime) { value.addProperty( - "createdTime", - Instant.ofEpochMilli(file.createdAtMillis).toString()); + "modifiedTime", + Instant.ofEpochMilli(file.modifiedAtMillis).toString()); } array.add(value); } @@ -1615,6 +1648,19 @@ private Response handleFileRead(Request request, URI uri, String id) { deletedFileIds.add(id); return Response.json(204, ""); } + if ("POST".equals(request.method) + && "PATCH".equals(request.headers.get("x-http-method-override"))) { + // files.update: merges appProperties and, like Drive, stamps modifiedTime. + JsonObject patch = readJson(request.body); + if (patch.has("appProperties")) { + for (Map.Entry entry : + patch.getAsJsonObject("appProperties").entrySet()) { + file.appProperties.put(entry.getKey(), entry.getValue().getAsString()); + } + } + file.modifiedAtMillis = serverNowMillis; + return Response.json(200, fileMetadata(file).toString(), file.eTag()); + } if ("media".equals(parseQuery(uri).get("alt"))) { mediaReads.merge(id, 1, Integer::sum); return Response.binary(200, file.content, file.eTag()); @@ -1774,6 +1820,8 @@ private void applyMetadata(DriveFile file, JsonObject metadata) { private DriveFile createFile(String name, String mimeType, String parentId) { DriveFile file = new DriveFile(Integer.toString(nextId.getAndIncrement()), name, mimeType); + file.createdAtMillis = serverNowMillis; + file.modifiedAtMillis = serverNowMillis; if (parentId != null) { file.parents.add(parentId); } @@ -1979,9 +2027,11 @@ public void close() { private static final class DriveFile { private final String id; - /** Drive's creation time; the fake's server clock is the tests' fixed CLOCK. */ + /** Drive's own timestamps, stamped from the fake server's clock. */ private long createdAtMillis = CLOCK.millis(); + private long modifiedAtMillis = CLOCK.millis(); + private byte[] content = new byte[0]; private String name; private String mimeType; diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index c58c6bf5..dce46333 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -375,6 +375,88 @@ public void encode_carriesAnAlternativeWhoseAttachmentIdMapsToDifferentContent() .isEqualTo(live.getCanonicalPayloadHash()); } + @Test + public void encode_keepsBothBlobsWhenOneNoteCarriesOneIdWithTwoContents() throws Exception { + // A duplicated block whose file was later replaced: one logical id, two hashes, in ONE + // record. Keying the re-keying by id sent both references to the second blob, so every + // receiver wrote its bytes for both and the first attachment's content was lost. + String otherHash = "0000000000000000000000000000000000000000000000000000000000000002"; + JsonObject payload = notePayload("Body", "image/png", 42L, "first.png"); + JsonObject second = + new SyncBundleCodec.AttachmentManifestEntry( + ATTACHMENT_ID, + otherHash, + "image/png", + 43L, + "attachments/" + otherHash, + "second.png") + .toJson(true); + payload.getAsJsonArray("attachmentsManifest").add(second); + payload.getAsJsonArray("attachmentHashes").add(otherHash); + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "second.png"); + payload.add("attachmentNames", names); + SyncRecord local = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + payload); + + SyncBundleCodec codec = new SyncBundleCodec(); + SyncBundleCodec.DecodedBundle decoded = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + java.util.Collections.singletonList(local)), + CREATED_AT))); + + assertThat(decoded.getAttachmentsByHash().keySet()).containsExactly(HASH, otherHash); + SyncRecord note = decoded.getSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID); + JsonArray manifest = note.getPayload().getAsJsonArray("attachmentsManifest"); + assertThat(manifest).hasSize(2); + assertThat(manifest.get(0).getAsJsonObject().get("sha256").getAsString()).isEqualTo(HASH); + assertThat(manifest.get(1).getAsJsonObject().get("sha256").getAsString()) + .isEqualTo(otherHash); + assertThat(manifest.get(0).getAsJsonObject().get("id").getAsString()) + .isEqualTo(ATTACHMENT_ID); + assertThat(manifest.get(1).getAsJsonObject().get("id").getAsString()) + .isEqualTo(ATTACHMENT_ID); + assertThat(note.getCanonicalPayloadHash()).isEqualTo(local.getCanonicalPayloadHash()); + } + + @Test + public void encode_refusesANoteThatReferencesOneManifestEntryTwice() throws Exception { + // The same id with the same content twice in one record cannot be told apart on the wire; + // the validator refuses it on the way out, as it now does on the way in, rather than let + // a receiver collapse two attachments into one. + JsonObject payload = notePayload("Body", "image/png", 42L, "first.png"); + payload.getAsJsonArray("attachmentsManifest") + .add( + payload.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .deepCopy()); + payload.getAsJsonArray("attachmentHashes").add(HASH); + SyncRecord local = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + payload); + + try { + new SyncBundleCodec() + .encode( + new SyncSnapshot(java.util.Collections.singletonList(local)), + CREATED_AT); + throw new AssertionError("Expected the duplicate reference to be refused"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("references one attachment twice"); + } + } + @Test public void decode_trimsADisplayNameSoEveryConsumerSeesTheNameThatWasValidated() throws Exception { diff --git a/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java b/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java new file mode 100644 index 00000000..fb77c8a4 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java @@ -0,0 +1,79 @@ +package com.pasich.mynotes.utils.backup.local; + +import static com.google.common.truth.Truth.assertThat; + +import com.pasich.mynotes.utils.constants.Backup; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Test; + +/** + * Whether a picked document is accepted as a backup. + * + *

Android's document picker appends " (2)" to a second backup saved into the same folder, so + * judging by the extension alone made the app refuse its own file — reproduced on a Pixel 7a. + */ +public class BackupFileValidatorTest { + + @Test + public void acceptsABackupArchiveWhateverThePickerCalledIt() throws Exception { + byte[] archive = archiveWith(Backup.FILE_NAME_BACKUP); + + assertThat( + BackupFileValidator.isAcceptable( + "My_Notes_Backup.mnbkn (2)", + () -> new ByteArrayInputStream(archive))) + .isTrue(); + assertThat( + BackupFileValidator.isAcceptable( + "anything at all", () -> new ByteArrayInputStream(archive))) + .isTrue(); + } + + @Test + public void stillAcceptsTheKnownExtensionsWithoutOpeningTheDocument() { + // The legacy non-archive formats can only be judged by name. + assertThat( + BackupFileValidator.isAcceptable( + "old.json", + () -> { + throw new AssertionError("must not open"); + })) + .isTrue(); + assertThat(BackupFileValidator.isAcceptable("Backup.MNBKN", () -> null)).isTrue(); + } + + @Test + public void refusesADocumentThatIsNeitherByNameNorByContent() throws Exception { + byte[] otherArchive = archiveWith("notes.txt"); + + assertThat( + BackupFileValidator.isAcceptable( + "photo.jpg", + () -> + new ByteArrayInputStream( + "not a zip".getBytes(StandardCharsets.UTF_8)))) + .isFalse(); + assertThat( + BackupFileValidator.isAcceptable( + "other.zip (2)", () -> new ByteArrayInputStream(otherArchive))) + .isFalse(); + assertThat(BackupFileValidator.isAcceptable("missing (2)", () -> null)).isFalse(); + } + + private static byte[] archiveWith(String entryName) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + zip.putNextEntry(new ZipEntry("attachments/note_1/photo.jpg")); + zip.write(new byte[] {1, 2, 3}); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry(entryName)); + zip.write("{}".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + return bytes.toByteArray(); + } +} From ce0ccbd0c65d20fbca6ae19639f25ab558347122 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Sat, 5 Sep 2026 11:24:00 +0300 Subject: [PATCH 4/4] fix(sync): accept the bundles 2.6.50 published, and ship as 2.6.51 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A narrow review of the previous commit found one release blocker. The within-note rule added there refused any note that referenced one manifest entry twice, but 2.6.50 — which is in closed testing now — legitimately publishes that shape when a note's column repeats a canonical id with the same content; its encoder only refused differing content. Because a read decodes every bundle in every root, one such bundle would have failed the whole read forever, and no read means no publish, so it could never be superseded or pruned either. The two shapes are now told apart. A repeated plain manifest id is one entry and one blob, so it is accepted and both references are kept. A repeated re-keyed id is the shape in which two attachments were collapsed into one reference; no released client produced it and it is still refused. Encode now gives each repeat its own wire id, so neither shape is emitted, including when a conflict alternative is republished. The prune grace compared Drive's timestamp against the phone's clock, so a device running fast collapsed it to zero. Both ends are Drive's clock now, taken from the Date header of the listing, which also drops the assumption that Drive bumps modifiedTime for an appProperties-only update. The backup content check ran on the main thread and inflated a wrongly picked archive to its end; it now runs on the background executor and gives up after 8 MiB. Ships as 2.6.51. The changelog keeps one entry for everything since 2.6.46, the last release users received: 2.6.50 reached closed testing only, so the difference between it and this build is repairs to code no one outside testing ever ran. 330 unit tests, 80 instrumentation tests, 0 failures; lint 0 errors. --- CHANGELOG.md | 2 +- app/build.gradle | 2 +- .../data/sync/GoogleDriveSyncBackend.java | 68 ++++++-- .../mynotes/data/sync/SyncBundleCodec.java | 24 ++- .../data/sync/SyncBundleValidator.java | 12 +- .../ui/view/activity/BackupActivity.java | 60 ++++--- .../backup/local/BackupFileValidator.java | 33 +++- .../data/sync/GoogleDriveSyncBackendTest.java | 77 ++++++++- .../data/sync/SyncBundleCodecTest.java | 148 ++++++++++++++++-- .../backup/local/BackupFileValidatorTest.java | 24 +++ 10 files changed, 390 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d052508..3afefd17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG -## [2.6.50] - 04.09.2026 +## [2.6.51] - 05.09.2026 **New** diff --git a/app/build.gradle b/app/build.gradle index d9d502d8..4ccc7e06 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -21,7 +21,7 @@ apply from: "$projectDir/gradle/libs-task.gradle" apply from: "$projectDir/gradle/changelog-task.gradle" -def appVersionCode = 50 +def appVersionCode = 51 def appVersionName = "2.6.${appVersionCode}" def gitCommitHashProvider = providers.exec { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index 838719b4..777a8bfa 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -47,9 +47,11 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final String PROPERTY_ATTACHMENT_SHA256 = "mynotesAttachmentSha256"; /** - * Set on a bundle the first time a read finds it outside the frontier. Drive stamps the update - * with its own {@code modifiedTime}, which is therefore the moment the bundle was seen to be - * superseded — measured by Drive's clock, not by whichever device happened to publish. + * Set on a bundle the first time a read finds it outside the frontier. Its value is Drive's own + * time at that moment, taken from the {@code Date} header of the listing that found it, so the + * grace period is measured on Drive's clock at both ends: the mark here, the current time from + * the latest response. Drive also stamps the update with {@code modifiedTime}, which serves as + * the fallback when a marking device had no server time to record. */ private static final String PROPERTY_BUNDLE_SUPERSEDED = "mynotesBundleSuperseded"; @@ -69,7 +71,8 @@ public final class GoogleDriveSyncBackend implements SyncBackend { * fetching the old bundle; an hour outlives any read, including the six-hourly worker's, which * WorkManager stops after ten minutes. Measured from the supersession mark, not from the * bundle's creation: a head created days ago and superseded seconds ago is exactly the file - * another device is most likely to be reading right now. + * another device is most likely to be reading right now. Both ends of the measurement are + * Drive's clock, see {@link #PROPERTY_BUNDLE_SUPERSEDED}. */ static final long BUNDLE_PRUNE_GRACE_MILLIS = 60L * 60L * 1000L; @@ -88,6 +91,15 @@ public final class GoogleDriveSyncBackend implements SyncBackend { /** Every bundle file the last read saw, so a publish can retire the ones it supersedes. */ private List lastReadBundles = Collections.emptyList(); + /** + * Drive's clock, as last reported in a response's {@code Date} header; zero until one arrives. + * + *

The one clock the prune grace is measured on. A phone running an hour fast would otherwise + * see every fresh supersession mark as an hour old and delete a bundle another device was still + * reading. + */ + private long lastDriveTimeMillis; + /** * The owned root folders, listed once per sync. * @@ -270,7 +282,13 @@ private void markSupersededBundles( } try { JsonObject patch = new JsonObject(); - patch.add("appProperties", appProperties(PROPERTY_BUNDLE_SUPERSEDED, "1")); + patch.add( + "appProperties", + appProperties( + PROPERTY_BUNDLE_SUPERSEDED, + lastDriveTimeMillis > 0L + ? Long.toString(lastDriveTimeMillis) + : "1")); // HttpURLConnection has no PATCH; Google's APIs honour the override header. HttpURLConnection connection = open("POST", apiBase + "/files/" + bundle.fileId + "?fields=id"); @@ -351,11 +369,14 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO * correctness. */ private void pruneSupersededBundles(@NonNull Collection frontierBundleIds) { - long cutoff = clock.millis() - BUNDLE_PRUNE_GRACE_MILLIS; + // Drive's clock against Drive's clock; the device clock is only the fallback when no + // response carried a Date header, which none of Google's do. + long now = lastDriveTimeMillis > 0L ? lastDriveTimeMillis : clock.millis(); + long cutoff = now - BUNDLE_PRUNE_GRACE_MILLIS; for (BundleFile bundle : lastReadBundles) { // Only a bundle that was already marked superseded when this sync read it, with the // mark's Drive-side timestamp older than the grace, may go. A bundle marked during - // this very read, or one whose mark Drive does not date, stays. + // this very read, or one whose mark carries no usable time, stays. if (frontierBundleIds.contains(bundle.logicalId) || bundle.supersededAtMillis == null || bundle.supersededAtMillis > cutoff) { @@ -711,18 +732,41 @@ private List findBundles(@NonNull String folderId) throws IOExceptio List result = new ArrayList<>(bundles.size()); for (int index = 0; index < bundles.size(); index++) { JsonObject file = bundles.get(index).getAsJsonObject(); - JsonObject properties = file.getAsJsonObject("appProperties"); - boolean marked = properties != null && properties.has(PROPERTY_BUNDLE_SUPERSEDED); result.add( new BundleFile( file.get("id").getAsString(), null, - marked ? instantOf(optionalString(file, "modifiedTime")) : null)); + supersededAtOf( + file.getAsJsonObject("appProperties"), + optionalString(file, "modifiedTime")))); } result.sort(Comparator.comparing(file -> file.fileId)); return result; } + /** + * When the bundle was marked superseded, on Drive's clock: the server time the marking device + * recorded in the property, else Drive's {@code modifiedTime} of that update; null when + * unmarked or when neither is usable. + */ + @Nullable + private static Long supersededAtOf( + @Nullable JsonObject appProperties, @Nullable String modifiedTime) { + if (appProperties == null || !appProperties.has(PROPERTY_BUNDLE_SUPERSEDED)) { + return null; + } + try { + long recorded = + Long.parseLong(appProperties.get(PROPERTY_BUNDLE_SUPERSEDED).getAsString()); + if (recorded > 1L) { + return recorded; + } + } catch (RuntimeException notATimestamp) { + // A marker without a time; Drive's own stamp of the update stands in. + } + return instantOf(modifiedTime); + } + /** Drive reports times as RFC 3339; anything else counts as unknown. */ @Nullable private static Long instantOf(@Nullable String time) { @@ -1411,6 +1455,10 @@ private JsonObject requestJsonIdempotent( @NonNull private JsonObject readJsonResponse(@NonNull HttpURLConnection connection) throws IOException { ensureSuccess(connection); + long serverTime = connection.getHeaderFieldDate("Date", 0L); + if (serverTime > 0L) { + lastDriveTimeMillis = serverTime; + } try (InputStream input = connection.getInputStream()) { return GSON.fromJson( new String( diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 8419981d..3920eef4 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -360,20 +360,29 @@ private static AttachmentPlan planAttachments( for (JsonElement element : manifestEntries) { AttachmentManifestEntry attachment = AttachmentManifestEntry.fromJson(element.getAsJsonObject()); + List recordWireIds = plan.wireIdsFor(record); String wireId = attachment.id; AttachmentManifestEntry sameId = plan.byWireId.get(wireId); if (sameId != null && !sameId.sameRemoteFile(attachment)) { - wireId = aliasFor(attachment); + wireId = aliasFor(attachment, 0); sameId = plan.byWireId.get(wireId); if (sameId != null && !sameId.sameRemoteFile(attachment)) { throw new IOException( "Two notes reference conflicting attachment metadata"); } } + // One record may repeat an entry — 2.6.50 published a duplicated block that way, + // and its receivers hold such manifests as conflict alternatives. Each repeat gets + // a wire id of its own, so a note never references one entry twice on the wire + // and every repeat maps back to the record's own id on the way in. + for (int occurrence = 1; recordWireIds.contains(wireId); occurrence++) { + wireId = aliasFor(attachment, occurrence); + sameId = plan.byWireId.get(wireId); + } if (sameId == null) { plan.byWireId.put(wireId, attachment.withId(wireId)); } - plan.wireIdsFor(record).add(wireId); + recordWireIds.add(wireId); AttachmentManifestEntry previous = seenByHash.putIfAbsent(attachment.sha256, attachment); if (previous != null && !previous.sameRemoteFile(attachment)) { @@ -389,11 +398,12 @@ private static AttachmentPlan planAttachments( /** Deterministic, so two devices publishing the same collision write the same bundle. */ @NonNull - private static String aliasFor(@NonNull AttachmentManifestEntry attachment) { - return UUID.nameUUIDFromBytes( - ("attachment-alias\n" + attachment.id + "\n" + attachment.sha256) - .getBytes(StandardCharsets.UTF_8)) - .toString(); + private static String aliasFor(@NonNull AttachmentManifestEntry attachment, int occurrence) { + String source = "attachment-alias\n" + attachment.id + "\n" + attachment.sha256; + if (occurrence > 0) { + source += "\n" + occurrence; + } + return UUID.nameUUIDFromBytes(source.getBytes(StandardCharsets.UTF_8)).toString(); } /** The manifest entries by wire id, and each record's wire ids in manifest order. */ diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index bdd36aad..21534111 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -176,17 +176,19 @@ private static void validateAttachmentReferences( } JsonObject attachmentNames = note.getAsJsonObject("attachmentNames"); validateAttachmentIdAliases(note, attachmentsById); + JsonObject aliases = note.getAsJsonObject(SyncBundleCodec.FIELD_ATTACHMENT_ID_ALIASES); Set withinNote = new LinkedHashSet<>(); for (JsonElement element : attachmentIds) { if (element == null || !element.isJsonPrimitive()) { throw new IOException("Sync note attachmentIds entry is invalid"); } String attachmentId = element.getAsString(); - if (!withinNote.add(attachmentId)) { - // Two references to one manifest entry can only mean two attachments were - // collapsed into one on the way out; accepting it made every receiver write - // one blob's bytes for both. - throw new IOException("Sync note references one attachment twice"); + if (!withinNote.add(attachmentId) && aliases != null && aliases.has(attachmentId)) { + // A plain id repeated is what 2.6.50 published for a duplicated block: two + // references, one blob, harmless, and it has to keep decoding. A re-keyed id + // repeated is something else: the record's two attachments were collapsed into + // one on the way out, and accepting it wrote one blob's bytes for both. + throw new IOException("Sync note collapses two attachments into one reference"); } SyncBundleCodec.AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); if (attachment == null) { diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 00897409..225879d6 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -111,25 +111,7 @@ public class BackupActivity extends BaseActivity if (result.getResultCode() == Activity.RESULT_OK) { if (result.getData() != null && result.getData().getData() != null) { Uri uri = result.getData().getData(); - - BackupFileValidator.isValidBackupFile( - this, - uri, - new BackupFileValidator.BackupValidatorCallback() { - @Override - public void onValid(String fileName) { - presenter.readFileBackupLocal(uri); - } - - @Override - public void onInvalid(String errorMessage) { - onInfoSnack( - errorMessage, - null, - SnackBarInfo.Error, - Snackbar.LENGTH_LONG); - } - }); + validatePickedBackup(uri); } } }); @@ -236,6 +218,46 @@ public void handleOnBackPressed() { }); } + /** + * Decides off the main thread whether the picked document is a backup. + * + *

When the name does not settle it the document is opened and inspected through the document + * provider, which on a wrong pick can mean inflating a large archive; on the main thread that + * froze the screen. The verdict is delivered back to the main thread. + */ + private void validatePickedBackup(@NonNull Uri uri) { + runInBackground( + () -> + BackupFileValidator.isValidBackupFile( + this, + uri, + new BackupFileValidator.BackupValidatorCallback() { + @Override + public void onValid(String fileName) { + runOnUiThread( + () -> { + if (!isFinishing() && !isDestroyed()) { + presenter.readFileBackupLocal(uri); + } + }); + } + + @Override + public void onInvalid(String errorMessage) { + runOnUiThread( + () -> { + if (!isFinishing() && !isDestroyed()) { + onInfoSnack( + errorMessage, + null, + SnackBarInfo.Error, + Snackbar.LENGTH_LONG); + } + }); + } + })); + } + /** The account tab draws itself from the state the next updateSyncUi() pushes. */ private void updateGoogleSignInButton() { updateSyncUi(); diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java index 43bc6d72..72a1606b 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java @@ -32,6 +32,15 @@ public class BackupFileValidator { private static final String EXT_ZIP = ".zip"; private static final String EXT_MNBK = ".mnbkn"; + /** + * How much of a picked archive is inflated looking for the backup JSON before giving up. + * + *

This app writes the JSON as the first entry, so the intended case costs a few kilobytes. A + * wrong pick that happens to be a large ZIP container — an APK, a document — would otherwise be + * inflated entry by entry to its end. + */ + static final long MAX_INSPECTED_BYTES = 8L * 1024L * 1024L; + /** Opens the picked document, so the content can be inspected. */ public interface ContentOpener { @Nullable @@ -41,6 +50,9 @@ public interface ContentOpener { /** * Validate a picked backup file. * + *

Reads the document when its name does not settle the question, so call it off the main + * thread; the callback is invoked on the calling thread. + * *

- If user cancels selection → return silently (no errors shown). - If filename cannot be * determined → callback.onInvalid(...) - If neither name nor content is a backup → * callback.onInvalid(...) - If valid → callback.onValid(filename) @@ -85,13 +97,32 @@ static boolean isAcceptable(@NonNull String name, @NonNull ContentOpener content /** True when the bytes are a ZIP archive holding the backup JSON entry. */ static boolean isBackupArchive(@NonNull InputStream input) throws IOException { + return isBackupArchive(input, MAX_INSPECTED_BYTES); + } + + /** + * @param maxInflatedBytes how much entry data may be inflated before the archive is judged not + * to be a backup; bounds the cost of a wrong pick. + */ + static boolean isBackupArchive(@NonNull InputStream input, long maxInflatedBytes) + throws IOException { try (ZipInputStream zip = new ZipInputStream(input)) { + byte[] scratch = new byte[8192]; + long inflated = 0L; ZipEntry entry; while ((entry = zip.getNextEntry()) != null) { if (Backup.FILE_NAME_BACKUP.equals(entry.getName())) { return true; } - zip.closeEntry(); + // Skipping an entry inflates it; count what that costs and stop rather than + // read a large container to its end. + int read; + while ((read = zip.read(scratch)) != -1) { + inflated += read; + if (inflated > maxInflatedBytes) { + return false; + } + } } } catch (RuntimeException notAnArchive) { // A ZipInputStream over arbitrary bytes throws on a malformed entry header. diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 01b0ff99..b41155d2 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -763,7 +763,9 @@ public void publish_retiresABundleOnlyOnceItHasBeenSupersededForTheWholeGrace() } @Test - public void publish_neverPrunesABundleWhoseSupersessionDriveDoesNotDate() throws Exception { + public void publish_measuresTheGraceOnDrivesClockNotThePhones() throws Exception { + // A phone running two hours fast used to see every fresh mark as two hours old and + // delete a bundle another device was still reading; Drive's Date header is the clock now. SyncBundleCodec codec = new SyncBundleCodec(); byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); @@ -772,8 +774,38 @@ public void publish_neverPrunesABundleWhoseSupersessionDriveDoesNotDate() throws snapshotWithTitle("Head"), CLOCK.instant(), Collections.singleton(baseId)); server.seedOwnedBundleBytes(base); server.seedOwnedBundleBytes(head); - server.withholdModifiedTime(); publish(backend(), snapshotWithTitle("Next")); + assertThat(server.supersededBundleCount()).isEqualTo(1); + + GoogleDriveSyncBackend fastPhone = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + Clock.offset(CLOCK, java.time.Duration.ofHours(2)), + new SyncBundleCodec()); + publish(fastPhone, snapshotWithTitle("Later")); + + // Drive's clock has not moved, so nothing has been superseded for the grace period. + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.bundleCount()).isEqualTo(4); + } + + @Test + public void publish_neverPrunesABundleWhoseSupersessionCarriesNoUsableTime() throws Exception { + // A bare marker written without server time, and Drive reporting no modifiedTime for + // the file: nothing can be proven old enough, so nothing goes. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] head = + codec.encode( + snapshotWithTitle("Head"), CLOCK.instant(), Collections.singleton(baseId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(head); + publish(backend(), snapshotWithTitle("Next")); + server.stripSupersessionTimes(); + server.withholdModifiedTime(); server.advanceClock(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); publish( @@ -785,11 +817,29 @@ public void publish_neverPrunesABundleWhoseSupersessionDriveDoesNotDate() throws new SyncBundleCodec()), snapshotWithTitle("Later")); - // Marked, but Drive reports no time for the mark: nothing can be proven old enough. assertThat(server.deletedFileIds()).isEmpty(); assertThat(server.bundleCount()).isEqualTo(4); } + @Test + public void readAttachment_fallsBackToTheGoodCopyInAnotherRoot() throws Exception { + // Duplicate roots are a supported state. With Drive's checksums withheld the bytes must + // be read to tell the copies apart; handing over the first root's only candidate unread + // let a corrupt copy there shadow the good one in the other root on every sync. + byte[] good = "good bytes".getBytes(StandardCharsets.UTF_8); + String hash = sha256(good); + server.seedCorruptAttachment(hash, "corrupt".getBytes(StandardCharsets.UTF_8)); + server.registerAttachment(good); + server.seedOwnedBundle(snapshot(NOTE_ID, hash)); + server.withholdChecksums(); + assertThat(server.ownedFolderCount()).isEqualTo(2); + + try (java.io.InputStream restored = backend().readAttachment(hash)) { + assertThat(restored).isNotNull(); + assertThat(readAll(restored)).isEqualTo(good); + } + } + // ------------------------------------------------- attachment transfer cost @Test @@ -1437,6 +1487,17 @@ void advanceClock(long millis) { serverNowMillis += millis; } + /** + * Turns every supersession mark into a bare marker, as a client without server time writes. + */ + void stripSupersessionTimes() { + for (DriveFile file : files.values()) { + if (file.appProperties.containsKey("mynotesBundleSuperseded")) { + file.appProperties.put("mynotesBundleSuperseded", "1"); + } + } + } + /** Whether the client has marked the newest {@code count} bundles as superseded. */ int supersededBundleCount() { int count = 0; @@ -1987,13 +2048,19 @@ private static String readLine(BufferedInputStream input) throws IOException { return output.toString(StandardCharsets.ISO_8859_1.name()); } - private static void writeResponse(OutputStream output, Response response) - throws IOException { + private void writeResponse(OutputStream output, Response response) throws IOException { + long serverNow = serverNowMillis; StringBuilder headers = new StringBuilder(); headers.append("HTTP/1.1 ").append(response.code).append(" OK\r\n"); headers.append("Content-Length: ").append(response.body.length).append("\r\n"); headers.append("Connection: close\r\n"); headers.append("Content-Type: ").append(response.contentType).append("\r\n"); + // Drive's own clock, as every Google response reports it. + headers.append("Date: ") + .append( + java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME.format( + Instant.ofEpochMilli(serverNow).atZone(ZoneOffset.UTC))) + .append("\r\n"); for (Map.Entry header : response.headers.entrySet()) { headers.append(header.getKey()) .append(": ") diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index dce46333..adc0980a 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -427,10 +427,12 @@ public void encode_keepsBothBlobsWhenOneNoteCarriesOneIdWithTwoContents() throws } @Test - public void encode_refusesANoteThatReferencesOneManifestEntryTwice() throws Exception { - // The same id with the same content twice in one record cannot be told apart on the wire; - // the validator refuses it on the way out, as it now does on the way in, rather than let - // a receiver collapse two attachments into one. + public void encode_republishesANoteThatRepeatsOneAttachmentWithoutCollapsingIt() + throws Exception { + // 2.6.50 receivers hold manifests with one entry repeated — a duplicated block whose file + // was not replaced — and republish them as conflict alternatives after the upgrade. + // Refusing the shape at encode failed every publish until the conflict was resolved; + // collapsing it changed the version's hash. Each repeat travels under its own wire id. JsonObject payload = notePayload("Body", "image/png", 42L, "first.png"); payload.getAsJsonArray("attachmentsManifest") .add( @@ -439,22 +441,146 @@ public void encode_refusesANoteThatReferencesOneManifestEntryTwice() throws Exce .getAsJsonObject() .deepCopy()); payload.getAsJsonArray("attachmentHashes").add(HASH); - SyncRecord local = + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "first.png"); + payload.add("attachmentNames", names); + SyncRecord repeated = SyncRecord.live( SyncRecord.Type.NOTE, NOTE_ID, Instant.parse("2026-08-31T12:00:01Z"), payload); + SyncRecord live = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:05Z"), + notePayload("Newer", "image/png", 42L, "first.png")); + + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(java.util.Collections.singletonList(live)), + CREATED_AT, + java.util.Collections.emptyList(), + java.util.Collections.singletonList(repeated), + java.util.Collections.emptySet()); + SyncBundleCodec.DecodedBundle decoded = codec.decode(new ByteArrayInputStream(bundle)); + + SyncRecord alternative = decoded.getAlternatives().get(0); + JsonArray manifest = alternative.getPayload().getAsJsonArray("attachmentsManifest"); + assertThat(manifest).hasSize(2); + assertThat(manifest.get(1).getAsJsonObject().get("id").getAsString()) + .isEqualTo(ATTACHMENT_ID); + assertThat(alternative.getCanonicalPayloadHash()) + .isEqualTo(repeated.getCanonicalPayloadHash()); + // Never the shape the validator has to refuse: one wire id, two references. + JsonObject records = + com.google.gson.JsonParser.parseString( + unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_RECORDS)) + .getAsJsonObject(); + JsonArray wireIds = + records.getAsJsonArray("alternatives") + .get(0) + .getAsJsonObject() + .getAsJsonArray("attachmentIds"); + assertThat(wireIds.get(0).getAsString()).isNotEqualTo(wireIds.get(1).getAsString()); + } + + @Test + public void decode_acceptsA2650BundleThatReferencesOneAttachmentTwice() throws Exception { + // 2.6.50 published attachmentIds [X, X] for a duplicated block with unchanged content, and + // such bundles sit on Drive in closed-testing accounts. A read decodes every bundle in + // every root, so refusing this shape failed every sync forever with no way to publish a + // successor that would let the bundle be pruned. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] valid = codec.encode(new SyncSnapshot(Arrays.asList(note("Milk"))), CREATED_AT); + JsonObject records = readRecords(valid); + JsonObject wireNote = records.getAsJsonArray("notes").get(0).getAsJsonObject(); + wireNote.getAsJsonArray("attachmentIds").add(ATTACHMENT_ID); + + SyncBundleCodec.DecodedBundle decoded = + codec.decode(new ByteArrayInputStream(rebuild(valid, records))); + + SyncRecord decodedNote = decoded.getSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID); + JsonArray manifest = decodedNote.getPayload().getAsJsonArray("attachmentsManifest"); + assertThat(manifest).hasSize(2); + assertThat(manifest.get(1).getAsJsonObject().get("sha256").getAsString()).isEqualTo(HASH); + // And it round-trips as the same version, so it never conflicts with itself. + SyncRecord again = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + java.util.Collections.singletonList( + decodedNote)), + CREATED_AT))) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + assertThat(again.getCanonicalPayloadHash()) + .isEqualTo(decodedNote.getCanonicalPayloadHash()); + } + + @Test + public void decode_refusesANoteWhoseRepeatedReferenceIsARekeyedOne() throws Exception { + // The shape an encoder produced when it collapsed two different blobs under one alias: + // the same re-keyed id twice. Only one blob is described, so the receiver cannot restore + // the note as it was; refusing is the only safe answer. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] valid = codec.encode(new SyncSnapshot(Arrays.asList(note("Milk"))), CREATED_AT); + JsonObject records = readRecords(valid); + JsonObject wireNote = records.getAsJsonArray("notes").get(0).getAsJsonObject(); + String alias = "550e8400-e29b-41d4-a716-446655440077"; + JsonArray ids = new JsonArray(); + ids.add(alias); + ids.add(alias); + wireNote.add("attachmentIds", ids); + JsonObject aliases = new JsonObject(); + aliases.addProperty(alias, ATTACHMENT_ID); + wireNote.add(SyncBundleCodec.FIELD_ATTACHMENT_ID_ALIASES, aliases); + JsonObject manifest = readManifest(valid); + manifest.getAsJsonArray("attachments").get(0).getAsJsonObject().addProperty("id", alias); try { - new SyncBundleCodec() - .encode( - new SyncSnapshot(java.util.Collections.singletonList(local)), - CREATED_AT); - throw new AssertionError("Expected the duplicate reference to be refused"); + codec.decode(new ByteArrayInputStream(rebuild(manifest, records))); + throw new AssertionError("Expected the collapsed shape to be refused"); } catch (IOException expected) { - assertThat(expected).hasMessageThat().contains("references one attachment twice"); + assertThat(expected).hasMessageThat().contains("collapses two attachments"); + } + } + + private static JsonObject readRecords(byte[] bundle) throws IOException { + return com.google.gson.JsonParser.parseString( + unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_RECORDS)) + .getAsJsonObject(); + } + + private static JsonObject readManifest(byte[] bundle) throws IOException { + return com.google.gson.JsonParser.parseString( + unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_MANIFEST)) + .getAsJsonObject(); + } + + /** Re-zips a bundle around edited records, refreshing the manifest checksum and length. */ + private static byte[] rebuild(byte[] bundle, JsonObject records) throws IOException { + return rebuild(readManifest(bundle), records); + } + + private static byte[] rebuild(JsonObject manifest, JsonObject records) throws IOException { + byte[] recordBytes = records.toString().getBytes(StandardCharsets.UTF_8); + manifest.addProperty("recordsSha256", SyncBundleValidator.sha256(recordBytes)); + manifest.addProperty("recordsBytes", recordBytes.length); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (java.util.zip.ZipOutputStream zip = + new java.util.zip.ZipOutputStream(output, StandardCharsets.UTF_8)) { + zip.putNextEntry(new ZipEntry(SyncBundleCodec.ENTRY_MANIFEST)); + zip.write(manifest.toString().getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry(SyncBundleCodec.ENTRY_RECORDS)); + zip.write(recordBytes); + zip.closeEntry(); } + return output.toByteArray(); } @Test diff --git a/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java b/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java index fb77c8a4..0df693b0 100644 --- a/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java @@ -64,6 +64,30 @@ public void refusesADocumentThatIsNeitherByNameNorByContent() throws Exception { assertThat(BackupFileValidator.isAcceptable("missing (2)", () -> null)).isFalse(); } + @Test + public void givesUpOnALargeContainerInsteadOfInflatingItToTheEnd() throws Exception { + // A wrong pick that happens to be a big ZIP container was inflated entry by entry, on the + // main thread, looking for the JSON; the check now stops once it has inflated its budget. + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + zip.putNextEntry(new ZipEntry("assets/big.bin")); + zip.write(new byte[512 * 1024]); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry(Backup.FILE_NAME_BACKUP)); + zip.write("{}".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + + assertThat( + BackupFileValidator.isBackupArchive( + new ByteArrayInputStream(bytes.toByteArray()), 64 * 1024)) + .isFalse(); + assertThat( + BackupFileValidator.isBackupArchive( + new ByteArrayInputStream(bytes.toByteArray()))) + .isTrue(); + } + private static byte[] archiveWith(String entryName) throws Exception { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ZipOutputStream zip = new ZipOutputStream(bytes)) {