Skip to content

HBASE-30334 Reopen parent regions on rollback of MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS #8583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nirdosh0110 wants to merge 3 commits into
base: master
Choose a base branch
from
+74 −1
Open

HBASE-30334 Reopen parent regions on rollback of MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS #8583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca1ad3b
HBASE-30334 Reopen parent regions on rollback of MERGE_TABLE_REGIONS_…
Sep 1, 2026
ca58f15
HBASE-30334 Add regression test for merge rollback after CHECK_CLOSED…
Sep 1, 2026
70ea371
Addressed comments 1
Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion ...r/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,9 +284,13 @@ protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegio
cleanupMergedRegion(env);
break;
case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
rollbackCloseRegionsForMerge(env);
break;

@Apache9 Apache9 Sep 1, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why we need this change? In general, when rolling back a procedure, finally we will arrive the MERGE_TABLE_REGIONS_CLOSE_REGIONS state and reopen the parent regions?

case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
rollbackCloseRegionsForMerge(env);
// No-op. The reopen TransitRegionStateProcedures for the two parents were already
// submitted by the rollback of MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS above;
// submitting a second batch here would race the first. Mirrors
// SplitTableRegionProcedure's rollback of SPLIT_TABLE_REGION_CLOSE_PARENT_REGION.
break;
case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
postRollBackMergeRegions(env);
Expand Down
69 changes: 69 additions & 0 deletions ...c/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
Expand All @@ -38,6 +40,7 @@
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
Expand All @@ -51,7 +54,10 @@
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -327,6 +333,69 @@ public void testRollbackAndDoubleExecution() throws Exception {
assertEquals(initialRegionCount, regions.size());
}

/**
* HBASE-30334 repro. Plant a stale recovered.edits file on a parent region so that
* MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS throws (the exact failure from the 49-min
* stuck-RIT incident). After rollback the parents MUST be OPEN. If they stay stuck
* (e.g. CLOSED / MERGING), we've reproduced the incident locally and the rollback path
* as-is is broken.
*/
@Test
public void testRollbackReopensParentsAfterCheckClosedRegionsFailure() throws Exception {
final TableName tableName = TableName.valueOf(testMethodName);
UTIL.createTable(tableName, new byte[][] { HConstants.CATALOG_FAMILY },
new byte[][] { new byte[] { 'b' } });
UTIL.waitUntilAllRegionsAssigned(tableName);

List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName);
assertEquals(2, ris.size());
RegionInfo[] regionsToMerge = new RegionInfo[] { ris.get(0), ris.get(1) };

Configuration conf = UTIL.getConfiguration();
Path regionDir =
FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), regionsToMerge[0]);
Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir);
FileSystem fs = CommonFSUtils.getRootDirFileSystem(conf);
fs.mkdirs(recoveredEditsDir);
Path staleFile = new Path(recoveredEditsDir, "0000000000000000001");
fs.createNewFile(staleFile);
assertTrue(WALSplitUtil.hasRecoveredEdits(conf, regionsToMerge[0]),
"stale recovered.edits file must be visible");

AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
LOG.info("HBASE-30334-DEBUG pre-merge: {} state={} / {} state={}",
regionsToMerge[0].getEncodedName(),
am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(),
regionsToMerge[1].getEncodedName(),
am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState());

final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
MergeTableRegionsProcedure proc =
new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
long procId = procExec.submitProcedure(proc);
ProcedureTestingUtility.waitProcedure(procExec, procId);

RegionState.State s0Post =
am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState();
RegionState.State s1Post =
am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState();
LOG.info("HBASE-30334-DEBUG post-rollback: {} state={} / {} state={}",
regionsToMerge[0].getEncodedName(), s0Post,
regionsToMerge[1].getEncodedName(), s1Post);

ProcedureTestingUtility.assertProcFailed(procExec, procId);

fs.delete(staleFile, false);

UTIL.waitFor(30_000, 500, () -> {
RegionState.State a =
am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState();
RegionState.State b =
am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState();
return a == RegionState.State.OPEN && b == RegionState.State.OPEN;
});
}

@Test
public void testMergeWithoutPONR() throws Exception {
final TableName tableName = TableName.valueOf("testMergeWithoutPONR");
Expand Down