diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java index da32446cd40e..c2a657e2970e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java @@ -26,6 +26,13 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_READ_BLACKLIST_USERS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_WORKERS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_WORKERS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION; @@ -34,6 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableSet; @@ -45,6 +53,7 @@ import org.apache.hadoop.ozone.om.KeyManagerImpl; import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.service.KeyLifecycleService; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -74,6 +83,7 @@ void reconfigurableProperties() { .add(OZONE_BLACKLIST_GROUPS) .add(OZONE_READ_BLACKLIST_USERS) .add(OZONE_READ_BLACKLIST_GROUPS) + .addAll(KeyLifecycleService.OZONE_KEY_LIFECYCLE_PROPERTIES) .build(); assertProperties(getSubject(), expected); @@ -260,4 +270,51 @@ void sstFilteringServiceInterval() throws ReconfigurationException { "SstFilteringService should not be null when enabled"); } + @Test + void keyLifecycleService() throws ReconfigurationException { + final OzoneManager om = cluster().getOzoneManager(); + final KeyLifecycleService service = om.getKeyManager().getKeyLifecycleService(); + final boolean originalEnabled = service.status().getIsEnabled(); + final String originalEnabledValue = om.getConfiguration().get(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED); + final String originalInterval = om.getConfiguration().get(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL); + final String originalWorkers = om.getConfiguration().get(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS); + + // All changed properties are applied together once reconfiguration completes + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, String.valueOf(!originalEnabled)); + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, "30s"); + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS, "3"); + assertEquals("30s", om.getConfiguration().get(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL)); + service.reconfigure(om.getConfiguration()); + assertEquals(!originalEnabled, service.status().getIsEnabled()); + assertEquals(!originalEnabled, om.getLifecycleServiceStatus().getIsEnabled()); + assertEquals(30000, service.getIntervalMillis()); + assertEquals(3, service.getExecutorService().getCorePoolSize()); + + // Invalid values are rejected and the configuration is left unchanged + assertThrows(ReconfigurationException.class, + () -> getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE, "0")); + assertThrows(ReconfigurationException.class, + () -> getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, "-1")); + assertEquals("30s", om.getConfiguration().get(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL)); + service.reconfigure(om.getConfiguration()); + assertEquals(30000, service.getIntervalMillis()); + + // Unset properties fall back to their defaults + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, null); + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, null); + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS, null); + assertNull(om.getConfiguration().get(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL)); + service.reconfigure(om.getConfiguration()); + assertEquals(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT, service.status().getIsEnabled()); + assertEquals(om.getConfiguration().getTimeDuration(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, + OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL_DEFAULT, MILLISECONDS), service.getIntervalMillis()); + assertEquals(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS_DEFAULT, service.getExecutorService().getCorePoolSize()); + + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, originalEnabledValue); + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, originalInterval); + getSubject().reconfigureProperty(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS, originalWorkers); + service.reconfigure(om.getConfiguration()); + assertEquals(originalEnabled, service.status().getIsEnabled()); + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index cce07fa8f613..8a49b3e24165 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -594,6 +594,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) .register(OZONE_BLACKLIST_GROUPS, this::reconfOzoneBlacklistGroups) .register(OZONE_READ_BLACKLIST_USERS, this::reconfOzoneReadBlacklistUsers) .register(OZONE_READ_BLACKLIST_GROUPS, this::reconfOzoneReadBlacklistGroups); + KeyLifecycleService.OZONE_KEY_LIFECYCLE_PROPERTIES.forEach(property -> + reconfigurationHandler.register(property, newVal -> validateKeyLifecycleProperty(property, newVal))); + reconfigurationHandler.registerCompleteCallback(this::reconfKeyLifecycleService); // Allow dynamic SCM node/address reconfiguration without restarting OM. // Address keys use prefix matching to discover new nodes added after startup. @@ -6257,6 +6260,30 @@ private String reconfOzoneThreadNumberDirDeletion(String newVal) { return newVal; } + private String validateKeyLifecycleProperty(String property, String newVal) { + OzoneConfiguration newConf = new OzoneConfiguration(getConfiguration()); + if (newVal == null) { + newConf.unset(property); + } else { + newConf.set(property, newVal); + } + KeyLifecycleService.validate(newConf); + return newVal; + } + + /** + * Apply all changed KeyLifecycleService properties at once, after each of them is validated and stored + * in the OM configuration. + */ + private void reconfKeyLifecycleService(Map changedKeys, Configuration newConf) { + KeyLifecycleService keyLifecycleService = getKeyManager().getKeyLifecycleService(); + // If the service is not running, the new values are picked up from the configuration when it is started. + if (keyLifecycleService != null + && changedKeys.keySet().stream().anyMatch(KeyLifecycleService.OZONE_KEY_LIFECYCLE_PROPERTIES::contains)) { + keyLifecycleService.reconfigure(getConfiguration()); + } + } + private String reconfOzoneSnapshotSSTFilteringServiceInterval(String newVal) { boolean wasSstFilteringSvcEnabled = ((KeyManagerImpl) keyManager).isSstFilteringSvcEnabled(); getConfiguration().set(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, newVal); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index 416e2d4115bf..07c66576f7e2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -26,6 +26,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK; @@ -34,10 +36,15 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_WORKERS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_WORKERS_DEFAULT; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.protobuf.ServiceException; import jakarta.annotation.Nullable; import java.io.IOException; @@ -120,11 +127,28 @@ public class KeyLifecycleService extends BackgroundService { public static final Logger LOG = LoggerFactory.getLogger(KeyLifecycleService.class); + /** + * Properties that can be changed at runtime via OM reconfiguration, see {@link #reconfigure}. + */ + public static final Set OZONE_KEY_LIFECYCLE_PROPERTIES = ImmutableSet.of( + OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, + OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, + OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT, + OZONE_KEY_LIFECYCLE_SERVICE_WORKERS, + OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE, + OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK, + OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, + OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED); + private final OzoneManager ozoneManager; - private int keyDeleteBatchSize; - private int mpuAbortLimitPerTask; - private long cachedDirMaxCount; + private volatile int keyDeleteBatchSize; + private volatile int mpuAbortLimitPerTask; + private volatile long cachedDirMaxCount; private final AtomicBoolean suspended; + // Set while the service is being restarted to apply a new interval, so that running tasks stop early. + private final AtomicBoolean restarting = new AtomicBoolean(false); private final AtomicBoolean isServiceEnabled; private final AtomicBoolean moveToTrashEnabled; private KeyLifecycleServiceMetrics metrics; @@ -132,8 +156,8 @@ public class KeyLifecycleService extends BackgroundService { private final ConcurrentHashMap inFlight; private OMMetadataManager omMetadataManager; private int ratisByteLimit; - private long stateSaveIntervalMs; - private long maxKeysProcessedPerState; + private volatile long stateSaveIntervalMs; + private volatile long maxKeysProcessedPerState; private ClientId clientId = ClientId.randomId(); private AtomicLong callId = new AtomicLong(0); private OzoneTrash ozoneTrash; @@ -238,7 +262,111 @@ private boolean shouldRun() { // OzoneManager can be null for testing return true; } - return isServiceEnabled.get() && !suspended.get() && getOzoneManager().isLeaderReady(); + return isServiceEnabled.get() && !suspended.get() && !restarting.get() && getOzoneManager().isLeaderReady(); + } + + /** + * Validate the values of {@link #OZONE_KEY_LIFECYCLE_PROPERTIES} in the given configuration. + * @throws IllegalArgumentException if any value is invalid + */ + public static void validate(ConfigurationSource conf) { + new ReconfigurableSettings(conf); + } + + /** + * Apply the values of all {@link #OZONE_KEY_LIFECYCLE_PROPERTIES} at once without restarting OM. + * Properties not set in the configuration fall back to their defaults. All changes except the interval take + * effect immediately, running tasks pick them up on their next batch scan. A new interval, or enabling a + * disabled service, restarts the service, running tasks stop at their next checkpoint and resume from the + * saved scan state in the next run. + * @param conf configuration holding the new values + * @throws IllegalArgumentException if any value is invalid, in which case nothing is changed + */ + public void reconfigure(ConfigurationSource conf) { + ReconfigurableSettings settings = new ReconfigurableSettings(conf); + keyDeleteBatchSize = settings.keyDeleteBatchSize; + mpuAbortLimitPerTask = settings.mpuAbortLimitPerTask; + cachedDirMaxCount = settings.cachedDirMaxCount; + stateSaveIntervalMs = settings.stateSaveIntervalMs; + maxKeysProcessedPerState = settings.maxKeysProcessedPerState; + moveToTrashEnabled.set(settings.moveToTrashEnabled); + setServiceTimeoutInNanos(settings.timeoutNanos); + setPoolSize(settings.workers); + boolean wasEnabled = isServiceEnabled.getAndSet(settings.enabled); + LOG.info("Reconfigured KeyLifecycleService: enabled = {}, interval = {} ms, timeout = {} ns, workers = {}, " + + "deleteBatchSize = {}, mpuAbortLimitPerTask = {}, cachedDirMaxCount = {}, stateSaveIntervalMs = {}, " + + "maxKeysProcessedPerState = {}, moveToTrashEnabled = {}", settings.enabled, settings.intervalMs, + settings.timeoutNanos, settings.workers, settings.keyDeleteBatchSize, settings.mpuAbortLimitPerTask, + settings.cachedDirMaxCount, settings.stateSaveIntervalMs, settings.maxKeysProcessedPerState, + settings.moveToTrashEnabled); + if (settings.intervalMs != getIntervalMillis() || (settings.enabled && !wasEnabled)) { + restart(settings.intervalMs); + } + } + + private void restart(long newIntervalMs) { + LOG.info("Restarting KeyLifecycleService with interval {} ms", newIntervalMs); + // Running and queued tasks check shouldRun() and stop early, which lets the executor drain + // instead of being force-terminated with tasks left in inFlight. + restarting.set(true); + try { + // shutdown() awaits the executor; do not hold this monitor (same object as + // BackgroundService.PeriodicalTask) or the pool thread can deadlock. + super.shutdown(); + } finally { + restarting.set(false); + } + synchronized (this) { + setInterval(newIntervalMs, TimeUnit.MILLISECONDS); + start(); + } + } + + /** + * Validated values of {@link #OZONE_KEY_LIFECYCLE_PROPERTIES}. + */ + private static final class ReconfigurableSettings { + private final boolean enabled; + private final long intervalMs; + private final long timeoutNanos; + private final int workers; + private final int keyDeleteBatchSize; + private final int mpuAbortLimitPerTask; + private final long cachedDirMaxCount; + private final long stateSaveIntervalMs; + private final long maxKeysProcessedPerState; + private final boolean moveToTrashEnabled; + + private ReconfigurableSettings(ConfigurationSource conf) { + enabled = conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT); + intervalMs = checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, conf.getTimeDuration( + OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL, OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS)); + timeoutNanos = checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT, conf.getTimeDuration( + OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT, OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT_DEFAULT, TimeUnit.NANOSECONDS)); + workers = (int) checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS, + conf.getInt(OZONE_KEY_LIFECYCLE_SERVICE_WORKERS, OZONE_KEY_LIFECYCLE_SERVICE_WORKERS_DEFAULT)); + keyDeleteBatchSize = (int) checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE, conf.getInt( + OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE, OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT)); + mpuAbortLimitPerTask = (int) checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK, + conf.getInt(OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK, + OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT)); + cachedDirMaxCount = checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT, + conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT, + OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT)); + stateSaveIntervalMs = checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, + conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT)); + maxKeysProcessedPerState = checkPositive(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, + conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT)); + moveToTrashEnabled = conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED, + OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT); + } + + private static long checkPositive(String property, long value) { + Preconditions.checkArgument(value > 0, property + " should be a positive value."); + return value; + } } public KeyLifecycleServiceMetrics getMetrics() { @@ -679,8 +807,8 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table while (!stack.isEmpty()) { if (!shouldRun()) { scanAborted = true; - LOG.info("KeyLifecycleService is suspended, disabled, or leader not ready. Stopping task for bucket {}.", - bucketName); + LOG.info("KeyLifecycleService is suspended, disabled, leader not ready, or about to restart. " + + "Stopping task for bucket {}.", bucketName); return; } @@ -1033,8 +1161,8 @@ private void evaluateBucket(OmBucketInfo bucketInfo, while (keyTblItr.hasNext()) { if (!shouldRun()) { scanAborted = true; - LOG.info("KeyLifecycleService is suspended, disabled, or leader not ready. Stopping task for bucket {}.", - bucketName); + LOG.info("KeyLifecycleService is suspended, disabled, leader not ready, or about to restart. " + + "Stopping task for bucket {}.", bucketName); return; } if (shouldSaveState()) { @@ -1099,8 +1227,8 @@ private void processMultipartUploads(OmBucketInfo bucketInfo, List rul while (mpuIterator.hasNext()) { if (!shouldRun()) { scanAborted = true; - LOG.info("KeyLifecycleService is suspended, disabled, or leader not ready. Stopping task for bucket {}.", - bucketName); + LOG.info("KeyLifecycleService is suspended, disabled, leader not ready, or about to restart. " + + "Stopping task for bucket {}.", bucketName); return; } Table.KeyValue entry = mpuIterator.next(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index ed51e2e46492..285cac5be9e7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -668,7 +668,8 @@ void testAbortedScanDoesNotMarkScanComplete(BucketLayout bucketLayout, boolean c assertNotNull(scanState); assertNull(scanState.getScanEndTime(), "Aborted scan must not persist scanEndTime so a new leader can resume"); - assertTrue(logCapturer.getOutput().contains("KeyLifecycleService is suspended, disabled, or leader not ready")); + assertTrue(logCapturer.getOutput() + .contains("KeyLifecycleService is suspended, disabled, leader not ready, or about to restart")); keyLifecycleService.resume(); deleteLifecyclePolicy(volumeName, bucketName); }