-
Notifications
You must be signed in to change notification settings - Fork 636
fix(store): throw on compaction-busy snapshot save; validate data/ on load (#3162) #3164
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
7ee5d42
f2b7219
8e121d4
46ce0c4
0e1c319
302844c
98fdaac
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,7 +24,6 @@ | |
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.concurrent.atomic.AtomicInteger; | ||
| import java.util.zip.Checksum; | ||
|
|
||
| import org.apache.commons.io.FileUtils; | ||
|
|
@@ -94,35 +93,41 @@ public void onSnapshotSave(final SnapshotWriter writer) throws HgStoreException | |
| final String snapshotDir = writer.getPath(); | ||
| if (partitionEngine != null) { | ||
| Integer groupId = partitionEngine.getGroupId(); | ||
| AtomicInteger state = businessHandler.getState(groupId); | ||
| if (state != null && state.get() == BusinessHandler.doing) { | ||
| return; | ||
| if (!businessHandler.tryLockCompactionRange(groupId)) { | ||
| throw new HgStoreException(HgStoreException.EC_RKDB_SNAPSHOT_SAVE_BUSY_FAIL, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
if (ret == RaftError.EIO.getNumber()) {
reportError(RaftError.EIO.getNumber(), "Fail to save snapshot.");
}
So a scheduled snapshot landing inside a Requested change: report busy with a code jraft does not escalate, |
||
| String.format( | ||
| "Partition %d snapshot save failed: " + | ||
| "compaction in progress", groupId)); | ||
| } | ||
| // rocks db snapshot | ||
| final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH; | ||
| businessHandler.saveSnapshot(graphSnapshotDir, "", groupId); | ||
|
|
||
| List<String> files = new ArrayList<>(); | ||
| File dir = new File(graphSnapshotDir); | ||
| File rootDirFile = new File(writer.getPath()); | ||
| // add all files in data dir | ||
| findFileList(dir, rootDirFile, files); | ||
|
|
||
| // load snapshot by learner ?? | ||
| for (String file : files) { | ||
| String checksum = calculateChecksum(writer.getPath() + File.separator + file); | ||
| if (checksum.length() != 0) { | ||
| LocalFileMetaOutter.LocalFileMeta meta = | ||
| LocalFileMetaOutter.LocalFileMeta.newBuilder() | ||
| .setChecksum(checksum) | ||
| .build(); | ||
| writer.addFile(file, meta); | ||
| } else { | ||
| writer.addFile(file); | ||
| try { | ||
| // rocks db snapshot | ||
| final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH; | ||
| businessHandler.saveSnapshot(graphSnapshotDir, "", groupId); | ||
|
|
||
| List<String> files = new ArrayList<>(); | ||
| File dir = new File(graphSnapshotDir); | ||
| File rootDirFile = new File(writer.getPath()); | ||
| // add all files in data dir | ||
| findFileList(dir, rootDirFile, files); | ||
|
|
||
| // load snapshot by learner ?? | ||
| for (String file : files) { | ||
| String checksum = calculateChecksum(writer.getPath() + File.separator + file); | ||
| if (checksum.length() != 0) { | ||
| LocalFileMetaOutter.LocalFileMeta meta = | ||
| LocalFileMetaOutter.LocalFileMeta.newBuilder() | ||
| .setChecksum(checksum) | ||
| .build(); | ||
| writer.addFile(file, meta); | ||
| } else { | ||
| writer.addFile(file); | ||
| } | ||
| } | ||
| // should_not_load wound not sync to learner | ||
| markShouldNotLoad(writer, true); | ||
| } finally { | ||
| businessHandler.unlockCompactionRange(groupId); | ||
| } | ||
| // should_not_load wound not sync to learner | ||
| markShouldNotLoad(writer, true); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -169,6 +174,15 @@ private String calculateChecksum(String path) { | |
| public void onSnapshotLoad(final SnapshotReader reader, long committedIndex) throws | ||
| HgStoreException { | ||
| final String snapshotDir = reader.getPath(); | ||
| final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH; | ||
|
|
||
| if (!new File(graphSnapshotDir).isDirectory()) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Placing this above the Line 188 returns early for a locally saved snapshot precisely because nothing is loaded from it. With the check above that return, such a snapshot missing The #3162 signature does not need the wider placement: the old early return in Requested change: move the block after the |
||
| throw new HgStoreException(HgStoreException.EC_RKDB_IMPORT_SNAPSHOT_FAIL, | ||
| String.format( | ||
| "Raft %d snapshot is corrupt, data dir %s is " + | ||
| "missing", partitionEngine.getGroupId(), | ||
| graphSnapshotDir)); | ||
| } | ||
|
|
||
| // No need to load locally saved snapshots | ||
| if (shouldNotLoad(reader)) { | ||
|
|
@@ -177,7 +191,6 @@ public void onSnapshotLoad(final SnapshotReader reader, long committedIndex) thr | |
| } | ||
|
|
||
| // Use snapshot directly | ||
| final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH; | ||
| log.info("Raft {} begin loadSnapshot, {}", partitionEngine.getGroupId(), graphSnapshotDir); | ||
| businessHandler.loadSnapshot(graphSnapshotDir, "", partitionEngine.getGroupId(), | ||
| committedIndex); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,11 +33,9 @@ public class HgStoreException extends RuntimeException { | |
| public static final int EC_RKDB_DOMERGE_FAIL = 1207; | ||
| public static final int EC_RKDB_DOGET_FAIL = 1208; | ||
| public static final int EC_RKDB_PD_FAIL = 1209; | ||
| public static final int EC_RKDB_TRUNCATE_FAIL = 1212; | ||
| public static final int EC_RKDB_EXPORT_SNAPSHOT_FAIL = 1214; | ||
| public static final int EC_RKDB_IMPORT_SNAPSHOT_FAIL = 1215; | ||
| public static final int EC_RKDB_TRANSFER_SNAPSHOT_FAIL = 1216; | ||
| public static final int EC_METRIC_FAIL = 1401; | ||
| public static final int EC_RKDB_SNAPSHOT_SAVE_BUSY_FAIL = 1217; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 This hunk also deletes three public constants, which is unrelated to the snapshot fix. Alongside adding Requested change: restore the three constants and keep this hunk to the single added code. If the cleanup is wanted, it belongs in its own PR. |
||
| private static final long serialVersionUID = 5193624480997934335L; | ||
| private final int code; | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,13 +17,14 @@ | |
|
|
||
| package org.apache.hugegraph.store.core; | ||
|
|
||
| import org.apache.hugegraph.store.core.snapshot.HgSnapshotHandlerTest; | ||
| import org.junit.runner.RunWith; | ||
| import org.junit.runners.Suite; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| // TODO: uncomment it until all test can run free. | ||
| //@RunWith(Suite.class) | ||
| //@Suite.SuiteClasses({ | ||
| // TODO: uncomment the rest of these classes once they can run free of each other. | ||
| // HgCmdClientTest.class, | ||
| // HgSnapshotHandlerTest.class, | ||
| // RaftUtilsTest.class, | ||
| // RaftOperationTest.class, | ||
| // UnsafeUtilTest.class, | ||
|
|
@@ -41,8 +42,10 @@ | |
| // PartitionInstructionProcessorTest.class, | ||
| // // Try to put it last | ||
| // HgBusinessImplTest.class | ||
| //}) | ||
|
|
||
| @RunWith(Suite.class) | ||
| @Suite.SuiteClasses({ | ||
| HgSnapshotHandlerTest.class | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The From job 101289530213, step 16: This is what the deleted Requested change: stop the two sharing a JVM. Move |
||
| }) | ||
| @Slf4j | ||
| public class CoreSuiteTest { | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🧹 The guard releases the monitor before the registration it is guarding.
raftRocksdbConfigRegisteredflips insidesynchronized (RaftRocksdbOptions.class)at line 66, but everyStorageOptionsFactory.register*call sits outside the block (lines 68-103, the calls at 81, 90 and 102). Because the flag is set before the work, a throw anywhere in the body,new LRUCache(SizeUnit.GB)on line 68 for instance, permanently suppresses every later attempt in that JVM with no log line, and the process then runs on jraft defaults instead of the configuredDBOptions/ColumnFamilyOptions. The same gap lets a concurrent second caller return early while the first is still registering, though today only test setups call this more than once.Requested change: hold the monitor across the whole method body, or set
raftRocksdbConfigRegistered = trueonly after the finalregisterRocksDBColumnFamilyOptionscall.