From ca1ad3ba3c8f9b9701c44020f4ea64c3acafb1f6 Mon Sep 17 00:00:00 2001 From: "nirdosh.yadav" <> Date: Tue, 1 Sep 2026 12:41:33 +0530 Subject: [PATCH 1/3] HBASE-30334 Reopen parent regions on rollback of MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS MergeTableRegionsProcedure.rollbackState() previously had a bare `break;` for the MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS case. When rollback fires at this state (e.g. checkClosedRegions() throws on a stale recovered.edits file), the parents were expected to be reopened on the next rollback tick through MERGE_TABLE_REGIONS_CLOSE_REGIONS. In production this cascade did not fire reliably: an OpenTSDB tsdb merge observed on cluster hbase31a on 2026-08-05 stayed RIT for 48 minutes until operator intervention, with no TRSP submitted for either parent during rollback. Mirror the pattern SplitTableRegionProcedure.rollbackState() already uses for the symmetric SPLIT_TABLE_REGIONS_CHECK_CLOSED_REGIONS state: call the reopen helper (rollbackCloseRegionsForMerge) directly on the failing state's own rollback body, and make the follow-on MERGE_TABLE_REGIONS_CLOSE_REGIONS an explicit no-op with the mirror comment from SplitTableRegionProcedure so duplicate TransitRegionStateProcedures are not submitted for parents already OPENING. The invariant is that any pre-PONR state that can fail owns its own rollback reopen, rather than relying on the framework cascading to the next state. --- .../hbase/master/assignment/MergeTableRegionsProcedure.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index f70099197deb..913cebf9fc08 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -284,9 +284,13 @@ protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegio cleanupMergedRegion(env); break; case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: + rollbackCloseRegionsForMerge(env); break; case MERGE_TABLE_REGIONS_CLOSE_REGIONS: - rollbackCloseRegionsForMerge(env); + // If it rolls back with state MERGE_TABLE_REGIONS_CLOSE_REGIONS, no need to call + // rollbackCloseRegionsForMerge(), otherwise, it will result in duplicate + // TransitRegionStateProcedures for parents that are already OPENING. Mirrors + // SplitTableRegionProcedure's rollback of SPLIT_TABLE_REGION_CLOSE_PARENT_REGION. break; case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: postRollBackMergeRegions(env); From ca58f15f8e2e07f4b4411850d8f281740ad6a90f Mon Sep 17 00:00:00 2001 From: "nirdosh.yadav" <> Date: Tue, 1 Sep 2026 12:41:40 +0530 Subject: [PATCH 2/3] HBASE-30334 Add regression test for merge rollback after CHECK_CLOSED_REGIONS failure Plants a stale recovered.edits file on a parent so that MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS throws (the exact trigger observed on the hbase31a stuck-RIT incident on 2026-08-05), submits a merge, and asserts both parents transition back to OPEN after rollback. Verifies the rollback body reopens the parents on the failing state's own tick, mirroring the pattern SplitTableRegionProcedure already exercises for SPLIT_TABLE_REGIONS_CHECK_CLOSED_REGIONS. --- .../TestMergeTableRegionsProcedure.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java index 1de06e82c02d..b822ee494c02 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java @@ -25,6 +25,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -38,6 +40,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; @@ -51,7 +54,10 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.wal.WALSplitUtil; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -327,6 +333,80 @@ public void testRollbackAndDoubleExecution() throws Exception { assertEquals(initialRegionCount, regions.size()); } + /** + * HBASE-30334 repro. Plant a stale recovered.edits file on a parent region so that + * MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS throws (the exact failure from the 49-min + * stuck-RIT incident). After rollback the parents MUST be OPEN. If they stay stuck + * (e.g. CLOSED / MERGING), we've reproduced the incident locally and the rollback path + * as-is is broken. + */ + @Test + public void testRollbackReopensParentsAfterCheckClosedRegionsFailure() throws Exception { + final TableName tableName = TableName.valueOf(testMethodName); + UTIL.createTable(tableName, new byte[][] { HConstants.CATALOG_FAMILY }, + new byte[][] { new byte[] { 'b' } }); + UTIL.waitUntilAllRegionsAssigned(tableName); + + List ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName); + assertEquals(2, ris.size()); + RegionInfo[] regionsToMerge = new RegionInfo[] { ris.get(0), ris.get(1) }; + + Configuration conf = UTIL.getConfiguration(); + Path regionDir = + FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), regionsToMerge[0]); + Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir); + FileSystem fs = CommonFSUtils.getRootDirFileSystem(conf); + fs.mkdirs(recoveredEditsDir); + Path staleFile = new Path(recoveredEditsDir, "0000000000000000001"); + fs.createNewFile(staleFile); + assertTrue(WALSplitUtil.hasRecoveredEdits(conf, regionsToMerge[0]), + "stale recovered.edits file must be visible"); + + AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); + LOG.info("HBASE-30334-DEBUG pre-merge: {} state={} / {} state={}", + regionsToMerge[0].getEncodedName(), + am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(), + regionsToMerge[1].getEncodedName(), + am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState()); + + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + MergeTableRegionsProcedure proc = + new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true); + long procId = procExec.submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(procExec, procId); + + RegionState.State s0Post = + am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(); + RegionState.State s1Post = + am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState(); + LOG.info("HBASE-30334-DEBUG post-rollback: {} state={} / {} state={}", + regionsToMerge[0].getEncodedName(), s0Post, + regionsToMerge[1].getEncodedName(), s1Post); + + ProcedureTestingUtility.assertProcFailed(procExec, procId); + + fs.delete(staleFile, false); + + long elapsed = org.apache.hadoop.hbase.Waiter.waitFor(conf, 30_000, 500, false, + new org.apache.hadoop.hbase.Waiter.Predicate() { + @Override + public boolean evaluate() { + RegionState.State a = + am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(); + RegionState.State b = + am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState(); + return a == RegionState.State.OPEN && b == RegionState.State.OPEN; + } + }); + + assertTrue(elapsed > 0, + "Parents must be OPEN after merge rollback (HBASE-30334). Final states: " + + regionsToMerge[0].getEncodedName() + "=" + + am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState() + ", " + + regionsToMerge[1].getEncodedName() + "=" + + am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState()); + } + @Test public void testMergeWithoutPONR() throws Exception { final TableName tableName = TableName.valueOf("testMergeWithoutPONR"); From 70ea3718cd9cec3c0aff0cc6bfd3b52a5965baa8 Mon Sep 17 00:00:00 2001 From: "nirdosh.yadav" <> Date: Tue, 1 Sep 2026 20:07:38 +0530 Subject: [PATCH 3/3] Addressed comments 1 --- .../MergeTableRegionsProcedure.java | 6 ++--- .../TestMergeTableRegionsProcedure.java | 25 ++++++------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 913cebf9fc08..fe6551e46cf3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -287,9 +287,9 @@ protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegio rollbackCloseRegionsForMerge(env); break; case MERGE_TABLE_REGIONS_CLOSE_REGIONS: - // If it rolls back with state MERGE_TABLE_REGIONS_CLOSE_REGIONS, no need to call - // rollbackCloseRegionsForMerge(), otherwise, it will result in duplicate - // TransitRegionStateProcedures for parents that are already OPENING. Mirrors + // No-op. The reopen TransitRegionStateProcedures for the two parents were already + // submitted by the rollback of MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS above; + // submitting a second batch here would race the first. Mirrors // SplitTableRegionProcedure's rollback of SPLIT_TABLE_REGION_CLOSE_PARENT_REGION. break; case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java index b822ee494c02..25da0f698f40 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java @@ -387,24 +387,13 @@ public void testRollbackReopensParentsAfterCheckClosedRegionsFailure() throws Ex fs.delete(staleFile, false); - long elapsed = org.apache.hadoop.hbase.Waiter.waitFor(conf, 30_000, 500, false, - new org.apache.hadoop.hbase.Waiter.Predicate() { - @Override - public boolean evaluate() { - RegionState.State a = - am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(); - RegionState.State b = - am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState(); - return a == RegionState.State.OPEN && b == RegionState.State.OPEN; - } - }); - - assertTrue(elapsed > 0, - "Parents must be OPEN after merge rollback (HBASE-30334). Final states: " - + regionsToMerge[0].getEncodedName() + "=" - + am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState() + ", " - + regionsToMerge[1].getEncodedName() + "=" - + am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState()); + UTIL.waitFor(30_000, 500, () -> { + RegionState.State a = + am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(); + RegionState.State b = + am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState(); + return a == RegionState.State.OPEN && b == RegionState.State.OPEN; + }); } @Test