diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index c5bf1ccad630..f1a53c471c9a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -97,6 +97,12 @@ public enum OzoneManagerVersion implements ComponentVersion { "OzoneManager version that gives each multipart upload part in an " + "encrypted bucket its own IV"), + RENAME_KEY_REJECTS_NON_FSO(106, + "OzoneManager version that rejects renameKey on buckets that are not " + + "FILE_SYSTEM_OPTIMIZED when the request sets requireFileSystemOptimized, " + + "so file system clients no longer need the client-side InfoBucket " + + "layout check for rename"), + UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" + " component, which may indicate it came from a component in a newer version"); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3ClientProtocol.java index d11d5fb7882e..ece97084d133 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3ClientProtocol.java @@ -394,6 +394,12 @@ public void renameKey(String volumeName, String bucketName, String fromKeyName, delegate.renameKey(volumeName, bucketName, fromKeyName, toKeyName); } + @Override + public void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName, + boolean requireFileSystemOptimized) throws IOException { + delegate.renameKey(volumeName, bucketName, fromKeyName, toKeyName, requireFileSystemOptimized); + } + @Override @Deprecated public void renameKeys(String volumeName, String bucketName, Map keyMap) throws IOException { diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 69824d0ca5b9..99b79ecda588 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -713,6 +713,18 @@ Map deleteKeys(String volumeName, String bucketName, void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName) throws IOException; + /** + * Renames an existing key within a bucket, like + * {@link #renameKey(String, String, String, String)}. + * + * @param requireFileSystemOptimized if true, OM rejects the rename with + * NOT_SUPPORTED_OPERATION unless the resolved bucket layout is + * FILE_SYSTEM_OPTIMIZED. An OM older than + * OzoneManagerVersion.RENAME_KEY_REJECTS_NON_FSO ignores it. + */ + void renameKey(String volumeName, String bucketName, String fromKeyName, + String toKeyName, boolean requireFileSystemOptimized) throws IOException; + /** * Renames existing keys within a bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 116edc2f99e3..2be84e739026 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1875,6 +1875,12 @@ public Map deleteKeys( @Override public void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName) throws IOException { + renameKey(volumeName, bucketName, fromKeyName, toKeyName, false); + } + + @Override + public void renameKey(String volumeName, String bucketName, + String fromKeyName, String toKeyName, boolean requireFileSystemOptimized) throws IOException { verifyVolumeName(volumeName); verifyBucketName(bucketName); if (checkKeyNameEnabled) { @@ -1885,6 +1891,7 @@ public void renameKey(String volumeName, String bucketName, .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(fromKeyName) + .setRequireFileSystemOptimized(requireFileSystemOptimized) .build(); ozoneManagerClient.renameKey(keyArgs, toKeyName); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index b408ad1ace64..60e642854e5f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@ -68,6 +68,9 @@ public final class OmKeyArgs extends WithMetadata implements Auditable { // Set by file system clients so that OM rejects a listing or a key info // read on an OBJECT_STORE bucket. private final boolean requireFileSystemLayout; + // Set by file system clients so that OM rejects a rename unless the bucket + // is FILE_SYSTEM_OPTIMIZED. + private final boolean requireFileSystemOptimized; private final String expectedETag; private final String excludedETag; private final boolean derivedKeyPiggyBacking; @@ -95,6 +98,7 @@ private OmKeyArgs(Builder b) { this.expectedDataGeneration = b.expectedDataGeneration; this.listPrefix = b.listPrefix; this.requireFileSystemLayout = b.requireFileSystemLayout; + this.requireFileSystemOptimized = b.requireFileSystemOptimized; this.expectedETag = b.expectedETag; this.excludedETag = b.excludedETag; this.derivedKeyPiggyBacking = b.derivedKeyPiggyBacking; @@ -200,6 +204,14 @@ public boolean isRequireFileSystemLayout() { return requireFileSystemLayout; } + /** + * Whether OM must reject a rename unless the resolved bucket layout is + * FILE_SYSTEM_OPTIMIZED, because the caller relies on atomic directory rename. + */ + public boolean isRequireFileSystemOptimized() { + return requireFileSystemOptimized; + } + public String getExpectedETag() { return expectedETag; } @@ -291,6 +303,7 @@ public static class Builder extends WithMetadata.Builder { private Long expectedDataGeneration = null; private String listPrefix = null; private boolean requireFileSystemLayout; + private boolean requireFileSystemOptimized; private String expectedETag; private String excludedETag; private boolean derivedKeyPiggyBacking; @@ -347,6 +360,7 @@ public Builder(OmKeyArgs obj) { this.acls = AclListBuilder.of(obj.acls); this.listPrefix = obj.listPrefix; this.requireFileSystemLayout = obj.requireFileSystemLayout; + this.requireFileSystemOptimized = obj.requireFileSystemOptimized; } public Builder setVolumeName(String volume) { @@ -495,6 +509,11 @@ public Builder setRequireFileSystemLayout(boolean require) { return this; } + public Builder setRequireFileSystemOptimized(boolean require) { + this.requireFileSystemOptimized = require; + return this; + } + public Builder setExpectedETag(String eTag) { this.expectedETag = eTag; return this; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 87eaea02418a..7186dc492e2c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -998,6 +998,9 @@ public void renameKey(OmKeyArgs args, String toKeyName) throws IOException { .setDataSize(args.getDataSize()).build(); req.setKeyArgs(keyArgs); req.setToKeyName(toKeyName); + if (args.isRequireFileSystemOptimized()) { + req.setRequireFileSystemOptimized(true); + } OMRequest omRequest = createOMRequest(Type.RenameKey) .setRenameKeyRequest(req) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index d1edd78db6a5..107fefad264d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -2029,6 +2029,45 @@ void testOpenInMissingBucketFails() throws Exception { assertEquals(BUCKET_NOT_FOUND, exception.getResult()); } + @Test + void testRenameDoesNotTriggerInfoBucketOnFso() throws Exception { + assumeTrue(bucketLayout.isFileSystemOptimized(), "Only a rename in an FSO bucket skips the bucket lookup"); + Path dir = new Path(bucketPath, "rename-rpc-" + RandomStringUtils.secure().nextAlphanumeric(5)); + Path src = new Path(dir, "file"); + Path dst = new Path(dir, "renamed"); + ContractTestUtils.touch(fs, src); + + OMMetrics metrics = getOMMetrics(); + long bucketInfosBefore = metrics.getNumBucketInfos(); + + assertTrue(fs.rename(src, dst)); + + assertEquals(bucketInfosBefore, metrics.getNumBucketInfos(), "an FSO rename must not trigger InfoBucket"); + assertTrue(fs.exists(dst)); + assertFalse(fs.exists(src)); + } + + @Test + void testRenameRejectsObsBucket() throws Exception { + // OM rejects the rename, and the client then rejects the bucket itself, as before. + OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); + DataTestUtil.createKey(obsBucket, "obs-key", "data".getBytes(StandardCharsets.UTF_8)); + Path obsBucketPath = new Path(new Path(OZONE_URI_DELIMITER, obsBucket.getVolumeName()), obsBucket.getName()); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> fs.rename(new Path(obsBucketPath, "obs-key"), new Path(obsBucketPath, "obs-key-2"))); + assertThat(exception.getMessage()).contains(obsBucket.getName()); + assertThat(exception.getMessage()).contains("OBJECT_STORE"); + } + + @Test + void testRenameInMissingBucketFails() throws Exception { + Path missingBucket = new Path(new Path(OZONE_URI_DELIMITER, volumeName), "missing-bucket"); + OMException exception = assertThrows(OMException.class, + () -> fs.rename(new Path(missingBucket, "file"), new Path(missingBucket, "file-2"))); + assertEquals(BUCKET_NOT_FOUND, exception.getResult()); + } + @Test void testGetFileStatus() throws Exception { String volumeNameLocal = getRandomNonExistVolumeName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsRenameRpcBenchmark.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsRenameRpcBenchmark.java new file mode 100644 index 000000000000..40e0e20845b0 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsRenameRpcBenchmark.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Benchmark for the OFS rename OM RPC count on FSO buckets (HDDS-16951). + * + *

The same workload runs unchanged on the baseline ({@code master}: InfoBucket + RenameKey per rename) and on the + * fix (a single RenameKey that asks OM to reject a bucket that is not FILE_SYSTEM_OPTIMIZED). It follows + * {@link TestOfsOpenRpcBenchmark}: each of {@link #NUM_BUCKETS} buckets holds one file, which is renamed back and + * forth between two names {@link #ACCESSES_PER_BUCKET} times, in a shuffled order, single-threaded and with + * {@link #CONCURRENT_THREADS} client threads. Each thread owns a disjoint set of buckets, so no two renames of the + * same file run at once and every rename succeeds. + */ +@Tag("benchmark") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class TestOfsRenameRpcBenchmark { + + private static final Logger LOG = LoggerFactory.getLogger(TestOfsRenameRpcBenchmark.class); + + /** Distinct buckets in the working set. */ + private static final int NUM_BUCKETS = 200; + /** rename calls per bucket. */ + private static final int ACCESSES_PER_BUCKET = 10; + private static final int TOTAL_CALLS = NUM_BUCKETS * ACCESSES_PER_BUCKET; + private static final long SHUFFLE_SEED = 20250831L; + /** Client threads issuing rename in the concurrent comparison; divides NUM_BUCKETS. */ + private static final int CONCURRENT_THREADS = 10; + + private MiniOzoneCluster cluster; + private OzoneClient client; + private OzoneConfiguration conf; + private String rootPath; + + /** Per bucket, the two names the file is renamed between, and which of them it has now. */ + private final Path[][] names = new Path[NUM_BUCKETS][2]; + private final int[] current = new int[NUM_BUCKETS]; + + @BeforeAll + void init() throws IOException, InterruptedException, TimeoutException { + conf = new OzoneConfiguration(); + conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.FILE_SYSTEM_OPTIMIZED.name()); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .build(); + cluster.waitForClusterToBeReady(); + client = cluster.newClient(); + rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY)); + + ObjectStore objectStore = client.getObjectStore(); + String volName = "benchvol"; + objectStore.createVolume(volName); + OzoneVolume volume = objectStore.getVolume(volName); + BucketArgs fsoArgs = BucketArgs.newBuilder().setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED).build(); + try (FileSystem setupFs = FileSystem.newInstance(URI.create(rootPath), conf)) { + for (int i = 0; i < NUM_BUCKETS; i++) { + String buckName = "benchbucket-" + i; + volume.createBucket(buckName, fsoArgs); + names[i][0] = new Path("/" + volName + "/" + buckName + "/dir/file-a"); + names[i][1] = new Path("/" + volName + "/" + buckName + "/dir/file-b"); + ContractTestUtils.touch(setupFs, names[i][0]); + } + assertThat(setupFs.exists(names[0][0])).isTrue(); + } + } + + @AfterAll + void shutdown() { + IOUtils.closeQuietly(client); + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + void benchmarkRename() throws Exception { + // Prime OM-side state and the JVM so the measured run pays no cold-start. + runWorkload(1); + + Result single = runWorkload(1); + Result concurrent = runWorkload(CONCURRENT_THREADS); + + assertWorkloadInvariants(single); + assertWorkloadInvariants(concurrent); + + logResult("single-threaded", single); + logResult(CONCURRENT_THREADS + " concurrent threads", concurrent); + } + + /** + * Invariants of the workload: every rename issues one RenameKey, and the bucket and volume RPCs stay within one per + * rename. + */ + private void assertWorkloadInvariants(Result r) { + assertThat(r.renameRpcs).isEqualTo(TOTAL_CALLS); + assertThat(r.bucketInfoRpcs).isBetween(0L, (long) TOTAL_CALLS); + assertThat(r.volumeInfoRpcs).isBetween(0L, (long) TOTAL_CALLS); + } + + private void logResult(String label, Result r) { + long omRpcs = r.volumeInfoRpcs + r.bucketInfoRpcs + r.renameRpcs; + LOG.info(String.format("%n" + + "BENCHRESULT OFS rename OM RPC benchmark (HDDS-16951) - %s%n" + + " buckets=%d, accesses/bucket=%d, total rename=%d, threads=%d%n" + + " ------------------------------------------------------------%n" + + " InfoVolume RPCs : %d%n" + + " InfoBucket RPCs : %d%n" + + " RenameKey RPCs : %d%n" + + " total OM RPCs : %d%n" + + " OM RPCs per call : %.2f%n" + + " ------------------------------------------------------------%n" + + " latency mean : %.3f ms%n" + + " latency p50 : %.3f ms%n" + + " latency p90 : %.3f ms%n" + + " latency p99 : %.3f ms%n" + + " latency max : %.3f ms%n" + + " throughput : %.1f rename/s%n", + label, NUM_BUCKETS, ACCESSES_PER_BUCKET, TOTAL_CALLS, r.threads, + r.volumeInfoRpcs, r.bucketInfoRpcs, r.renameRpcs, + omRpcs, omRpcs / (double) TOTAL_CALLS, + millis(r.mean()), millis(r.percentile(0.50)), + millis(r.percentile(0.90)), millis(r.percentile(0.99)), + millis(r.max()), r.opsPerSecond())); + } + + private static double millis(double nanos) { + return nanos / (double) TimeUnit.MILLISECONDS.toNanos(1); + } + + /** + * The bucket sequence of each thread: the buckets {@code b} with {@code b % threads == t}, each + * {@link #ACCESSES_PER_BUCKET} times, shuffled. + */ + private static List> bucketSequences(int threads) { + Random random = new Random(SHUFFLE_SEED); + List> sequences = new ArrayList<>(threads); + for (int t = 0; t < threads; t++) { + List sequence = new ArrayList<>(); + for (int b = t; b < NUM_BUCKETS; b += threads) { + for (int a = 0; a < ACCESSES_PER_BUCKET; a++) { + sequence.add(b); + } + } + Collections.shuffle(sequence, random); + sequences.add(sequence); + } + return sequences; + } + + /** Renames the file of the bucket to its other name; only the thread that owns the bucket calls this. */ + private void renameOnce(FileSystem fs, int bucket) throws IOException { + int from = current[bucket]; + if (!fs.rename(names[bucket][from], names[bucket][1 - from])) { + throw new IOException("rename failed: " + names[bucket][from]); + } + current[bucket] = 1 - from; + } + + private Result runWorkload(int threads) throws IOException, InterruptedException { + OzoneConfiguration runConf = new OzoneConfiguration(conf); + runConf.set(FS_DEFAULT_NAME_KEY, rootPath); + + OMMetrics metrics = cluster.getOzoneManager().getMetrics(); + List> sequences = bucketSequences(threads); + long[] latencyNanos = new long[TOTAL_CALLS]; + // A new FileSystem instance per run, so each run starts with no client-side state carried over. + try (FileSystem fs = FileSystem.newInstance(URI.create(rootPath), runConf)) { + long volumeInfosBefore = metrics.getNumVolumeInfos(); + long bucketInfosBefore = metrics.getNumBucketInfos(); + long renamesBefore = metrics.getNumKeyRenames(); + long elapsedNanos = runThreads(fs, sequences, latencyNanos); + return new Result(threads, + metrics.getNumVolumeInfos() - volumeInfosBefore, + metrics.getNumBucketInfos() - bucketInfosBefore, + metrics.getNumKeyRenames() - renamesBefore, + elapsedNanos, latencyNanos); + } + } + + private long runThreads(FileSystem fs, List> sequences, long[] latencyNanos) + throws IOException, InterruptedException { + int threads = sequences.size(); + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch startGate = new CountDownLatch(1); + CountDownLatch doneGate = new CountDownLatch(threads); + AtomicReference failure = new AtomicReference<>(); + // Each thread writes only its own contiguous range of latency indices, so no synchronization is needed per call. + int offset = 0; + for (List sequence : sequences) { + final int from = offset; + offset += sequence.size(); + pool.submit(() -> { + try { + startGate.await(); + int i = from; + for (int bucket : sequence) { + long callStart = System.nanoTime(); + renameOnce(fs, bucket); + latencyNanos[i++] = System.nanoTime() - callStart; + } + } catch (IOException e) { + failure.compareAndSet(null, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + doneGate.countDown(); + } + }); + } + long startNanos = System.nanoTime(); + startGate.countDown(); + doneGate.await(); + long elapsedNanos = System.nanoTime() - startNanos; + pool.shutdownNow(); + if (failure.get() != null) { + throw failure.get(); + } + return elapsedNanos; + } + + private static final class Result { + private final int threads; + private final long volumeInfoRpcs; + private final long bucketInfoRpcs; + private final long renameRpcs; + private final long elapsedNanos; + private final long[] sortedLatencyNanos; + + Result(int threads, long volumeInfoRpcs, long bucketInfoRpcs, long renameRpcs, long elapsedNanos, + long[] latencyNanos) { + this.threads = threads; + this.volumeInfoRpcs = volumeInfoRpcs; + this.bucketInfoRpcs = bucketInfoRpcs; + this.renameRpcs = renameRpcs; + this.elapsedNanos = elapsedNanos; + this.sortedLatencyNanos = latencyNanos.clone(); + Arrays.sort(this.sortedLatencyNanos); + } + + double opsPerSecond() { + return TOTAL_CALLS / (elapsedNanos / (double) TimeUnit.SECONDS.toNanos(1)); + } + + double mean() { + long sum = 0; + for (long l : sortedLatencyNanos) { + sum += l; + } + return sum / (double) sortedLatencyNanos.length; + } + + double percentile(double q) { + int idx = (int) Math.ceil(q * sortedLatencyNanos.length) - 1; + idx = Math.max(0, Math.min(sortedLatencyNanos.length - 1, idx)); + return sortedLatencyNanos[idx]; + } + + double max() { + return sortedLatencyNanos[sortedLatencyNanos.length - 1]; + } + } +} diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index ff7b325b703c..b38c7331d6df 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1480,6 +1480,10 @@ message RenameKeyRequest{ required KeyArgs keyArgs = 1; required string toKeyName = 2; optional uint64 updateID = 3; + // Set by file system clients that rely on atomic directory rename: OM + // rejects the request with NOT_SUPPORTED_OPERATION unless the resolved + // bucket layout is FILE_SYSTEM_OPTIMIZED. + optional bool requireFileSystemOptimized = 4; } message RenameKeyResponse{ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index 40c493ee3383..2f1b56297b7c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -76,6 +76,14 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .getRenameKeyRequest(); Objects.requireNonNull(renameKeyRequest, "renameKeyRequest == null"); + // Checked before Ratis only: preExecute does not run when the request is applied. The request was created for the + // link-resolved layout, and the OBJECT_STORE request class also serves LEGACY, so test the layout itself. + if (renameKeyRequest.getRequireFileSystemOptimized() && !getBucketLayout().isFileSystemOptimized()) { + throw new OMException("Bucket: " + renameKeyRequest.getKeyArgs().getBucketName() + + " has layout: " + getBucketLayout() + ", but the rename requires " + + BucketLayout.FILE_SYSTEM_OPTIMIZED + ".", OMException.ResultCodes.NOT_SUPPORTED_OPERATION); + } + KeyArgs renameKeyArgs = renameKeyRequest.getKeyArgs(); if (ozoneManager.getConfig().isKeyNameCharacterCheckEnabled()) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequest.java index 72b621e46af8..4cf89f534d15 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequest.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OMAuditLogger; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -46,6 +47,8 @@ import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.ArgumentCaptor; /** @@ -108,6 +111,44 @@ public void testPreExecute() throws Exception { volumeName, bucketName, fromKeyName, toKeyName)); } + /** A rename that requires a FILE_SYSTEM_OPTIMIZED bucket passes preExecute only on one. */ + @Test + public void testPreExecuteRequireFileSystemOptimized() throws Exception { + addKeyToTable(fromKeyInfo); + OMRequest request = requireFileSystemOptimized( + createRenameKeyRequest(volumeName, bucketName, fromKeyName, toKeyName), true); + if (getBucketLayout().isFileSystemOptimized()) { + assertThat(doPreExecute(request).getRenameKeyRequest().getRequireFileSystemOptimized()).isTrue(); + } else { + OMException e = assertThrows(OMException.class, () -> getOMKeyRenameRequest(request).preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION, e.getResult()); + assertThat(e.getMessage()).contains(bucketName).contains(getBucketLayout().name()); + } + } + + /** + * The OBJECT_STORE request class also serves LEGACY: with the field set, both layouts are rejected before Ratis; + * without it, both are served as before, as for the shell, trash and lifecycle callers. + */ + @ParameterizedTest + @EnumSource(value = BucketLayout.class, names = {"LEGACY", "OBJECT_STORE"}) + public void testPreExecuteNonFileSystemOptimizedLayouts(BucketLayout layout) throws Exception { + OMRequest request = createRenameKeyRequest(volumeName, bucketName, fromKeyName, toKeyName); + + OMException e = assertThrows(OMException.class, + () -> new OMKeyRenameRequest(requireFileSystemOptimized(request, true), layout).preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION, e.getResult()); + + OMRequest served = new OMKeyRenameRequest(requireFileSystemOptimized(request, false), layout) + .preExecute(ozoneManager); + assertThat(served.getRenameKeyRequest().getRequireFileSystemOptimized()).isFalse(); + } + + private static OMRequest requireFileSystemOptimized(OMRequest request, boolean require) { + return request.toBuilder().setRenameKeyRequest( + request.getRenameKeyRequest().toBuilder().setRequireFileSystemOptimized(require)).build(); + } + @Test public void testValidateAndUpdateCache() throws Exception { String dbFromKey = addKeyToTable(fromKeyInfo); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index cb0832ee7692..9871fc313be0 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -537,6 +537,27 @@ void rename(OzoneBucket bucket, String path, String newPath) bucket.renameKey(key, newKey); } + /** + * Whether OM rejects a rename that requires a FILE_SYSTEM_OPTIMIZED bucket on any other layout (HDDS-16951), so a + * file system rename can be sent without fetching the bucket to learn its layout. + */ + boolean isRenameRequiringFileSystemOptimizedSupported() { + return proxy.getOmVersion().compareTo(OzoneManagerVersion.RENAME_KEY_REJECTS_NON_FSO) >= 0; + } + + /** + * Like {@link #rename(OzoneBucket, String, String)} without a fetched bucket: one RenameKey that OM accepts only + * on a FILE_SYSTEM_OPTIMIZED bucket, and rejects with NOT_SUPPORTED_OPERATION on any other layout. + */ + void renameRequiringFileSystemOptimized(String path, String newPath) throws IOException { + OFSPath ofsPath = new OFSPath(path, config); + OFSPath ofsNewPath = new OFSPath(newPath, config); + // No same-bucket policy check here since this call path is controlled + proxy.renameKey(ofsPath.getVolumeName(), ofsPath.getBucketName(), ofsPath.getKeyName(), + ofsNewPath.getKeyName(), true); + incrementCounter(Statistic.OBJECTS_RENAMED, 1); + } + /** * Helper method to create an directory specified by key name in bucket. * diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 53c521b972d9..79059b531088 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -53,6 +53,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Objects; +import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; @@ -433,6 +434,13 @@ private boolean renameInSpan(Path src, Path dst) throws IOException { throw new IOException("Cannot rename across different volumes or buckets." + " Only supported within the same bucket."); } + // An empty bucket name takes the getBucket path, which reports it as FileNotFoundException. + if (!ofsSrc.getBucketName().isEmpty() && adapterImpl.isRenameRequiringFileSystemOptimizedSupported()) { + Optional renamed = renameIfFileSystemOptimized(ofsSrc, ofsDst); + if (renamed.isPresent()) { + return renamed.get(); + } + } OzoneBucket bucket = adapterImpl.getBucket(ofsSrc, false); if (bucket.getBucketLayout().isFileSystemOptimized()) { return renameFSO(bucket, ofsSrc, ofsDst); @@ -517,26 +525,57 @@ private boolean renameInSpan(Path src, Path dst) throws IOException { private boolean renameFSO(OzoneBucket bucket, OFSPath srcPath, OFSPath dstPath) throws IOException { // construct src and dst key paths - String srcKeyPath = srcPath.getNonKeyPathNoPrefixDelim() + - OZONE_URI_DELIMITER + srcPath.getKeyName(); - String dstKeyPath = dstPath.getNonKeyPathNoPrefixDelim() + - OZONE_URI_DELIMITER + dstPath.getKeyName(); + String srcKeyPath = fsoRenamePath(srcPath); + String dstKeyPath = fsoRenamePath(dstPath); try { adapterImpl.rename(bucket, srcKeyPath, dstKeyPath); } catch (OMException ome) { - LOG.error("rename key failed: {}. source:{}, destin:{}", - ome.getMessage(), srcKeyPath, dstKeyPath); - if (OMException.ResultCodes.KEY_ALREADY_EXISTS == ome.getResult() || - OMException.ResultCodes.KEY_RENAME_ERROR == ome.getResult() || - OMException.ResultCodes.KEY_NOT_FOUND == ome.getResult()) { - return false; - } else { - throw ome; - } + return fsoRenameFailed(ome, srcKeyPath, dstKeyPath); } return true; } + /** + * Like {@link #renameFSO} without fetching the bucket first: one RenameKey that OM accepts only on a + * FILE_SYSTEM_OPTIMIZED bucket (HDDS-16951), so an FSO rename needs no InfoBucket. Returns empty when the rename + * must take the path that fetches the bucket instead: OM rejected the bucket as not FILE_SYSTEM_OPTIMIZED, or the + * volume or bucket does not exist, which that path reports as before. + */ + private Optional renameIfFileSystemOptimized(OFSPath srcPath, OFSPath dstPath) throws IOException { + String srcKeyPath = fsoRenamePath(srcPath); + String dstKeyPath = fsoRenamePath(dstPath); + try { + adapterImpl.renameRequiringFileSystemOptimized(srcKeyPath, dstKeyPath); + } catch (OMException ome) { + switch (ome.getResult()) { + case NOT_SUPPORTED_OPERATION: + case BUCKET_NOT_FOUND: + case VOLUME_NOT_FOUND: + return Optional.empty(); + default: + return Optional.of(fsoRenameFailed(ome, srcKeyPath, dstKeyPath)); + } + } + return Optional.of(true); + } + + private static String fsoRenamePath(OFSPath path) { + return path.getNonKeyPathNoPrefixDelim() + OZONE_URI_DELIMITER + path.getKeyName(); + } + + /** Returns false for the failures that make an FSO rename return false, otherwise rethrows. */ + private static boolean fsoRenameFailed(OMException ome, String srcKeyPath, String dstKeyPath) throws OMException { + LOG.error("rename key failed: {}. source:{}, destin:{}", + ome.getMessage(), srcKeyPath, dstKeyPath); + if (OMException.ResultCodes.KEY_ALREADY_EXISTS == ome.getResult() || + OMException.ResultCodes.KEY_RENAME_ERROR == ome.getResult() || + OMException.ResultCodes.KEY_NOT_FOUND == ome.getResult()) { + return false; + } else { + throw ome; + } + } + /** * Intercept rename to trash calls from TrashPolicyDefault. */ diff --git a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemRename.java b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemRename.java new file mode 100644 index 000000000000..a7dff36ba8eb --- /dev/null +++ b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemRename.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URI; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OFSPath; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +/** + * Unit tests for {@link BasicRootedOzoneFileSystem#rename} on an OM that rejects a rename requiring a + * FILE_SYSTEM_OPTIMIZED bucket on other layouts (HDDS-16951): an FSO rename is one RenameKey with no InfoBucket, and + * every other case falls back to the path that fetches the bucket. Uses a mock adapter so no cluster is required. + */ +public class TestRootedOzoneFileSystemRename { + + private static final Path SRC = new Path("/vol/bucket/dir/file"); + private static final Path DST = new Path("/vol/bucket/dir/file2"); + private static final String SRC_KEY_PATH = "vol/bucket/dir/file"; + private static final String DST_KEY_PATH = "vol/bucket/dir/file2"; + + private BasicRootedOzoneClientAdapterImpl adapter; + private BasicRootedOzoneFileSystem fs; + private OzoneBucket bucket; + + /** Test FS that injects a mock adapter instead of connecting to OM. */ + private final class MockAdapterFs extends BasicRootedOzoneFileSystem { + @Override + protected OzoneClientAdapter createAdapter(ConfigurationSource conf, String omHost, int omPort) { + return adapter; + } + } + + @BeforeEach + public void setUp() throws IOException { + adapter = mock(BasicRootedOzoneClientAdapterImpl.class); + fs = new MockAdapterFs(); + fs.initialize(URI.create("ofs://om/"), new OzoneConfiguration()); + bucket = mock(OzoneBucket.class); + when(adapter.getBucket(any(OFSPath.class), anyBoolean())).thenReturn(bucket); + when(adapter.isRenameRequiringFileSystemOptimizedSupported()).thenReturn(true); + } + + private void failFsoRename(ResultCodes code) throws IOException { + doThrow(new OMException("failed", code)).when(adapter).renameRequiringFileSystemOptimized(anyString(), anyString()); + } + + @Test + public void fsoRenameIsOneRenameKeyWithoutInfoBucket() throws IOException { + assertTrue(fs.rename(SRC, DST)); + + verify(adapter).renameRequiringFileSystemOptimized(SRC_KEY_PATH, DST_KEY_PATH); + verify(adapter, never()).getBucket(any(OFSPath.class), anyBoolean()); + verify(adapter, never()).rename(any(OzoneBucket.class), anyString(), anyString()); + } + + @ParameterizedTest + @EnumSource(value = ResultCodes.class, names = {"KEY_ALREADY_EXISTS", "KEY_RENAME_ERROR", "KEY_NOT_FOUND"}) + public void fsoRenameFailureReturnsFalse(ResultCodes code) throws IOException { + failFsoRename(code); + + assertFalse(fs.rename(SRC, DST)); + verify(adapter, never()).getBucket(any(OFSPath.class), anyBoolean()); + } + + @Test + public void otherFsoRenameErrorIsRethrown() throws IOException { + failFsoRename(ResultCodes.PERMISSION_DENIED); + + OMException e = assertThrows(OMException.class, () -> fs.rename(SRC, DST)); + assertEquals(ResultCodes.PERMISSION_DENIED, e.getResult()); + verify(adapter, never()).getBucket(any(OFSPath.class), anyBoolean()); + } + + @Test + public void legacyBucketFallsBackToTheIteratingRename() throws IOException { + failFsoRename(ResultCodes.NOT_SUPPORTED_OPERATION); + when(bucket.getBucketLayout()).thenReturn(BucketLayout.LEGACY); + // The LEGACY path starts by looking up the source. + when(adapter.getFileStatus(anyString(), any(URI.class), any(Path.class), anyString(), anyBoolean())) + .thenThrow(new FileNotFoundException("no source")); + + assertFalse(fs.rename(SRC, DST)); + verify(adapter).getBucket(any(OFSPath.class), anyBoolean()); + verify(adapter).getFileStatus(anyString(), any(URI.class), any(Path.class), anyString(), anyBoolean()); + verify(adapter, never()).rename(any(OzoneBucket.class), anyString(), anyString()); + } + + @Test + public void objectStoreBucketFallsBackToTheClientSideRejection() throws IOException { + failFsoRename(ResultCodes.NOT_SUPPORTED_OPERATION); + IllegalArgumentException rejection = new IllegalArgumentException("Bucket: bucket has layout: OBJECT_STORE"); + when(adapter.getBucket(any(OFSPath.class), anyBoolean())).thenThrow(rejection); + + assertSame(rejection, assertThrows(IllegalArgumentException.class, () -> fs.rename(SRC, DST))); + } + + @ParameterizedTest + @EnumSource(value = ResultCodes.class, names = {"BUCKET_NOT_FOUND", "VOLUME_NOT_FOUND"}) + public void missingBucketFailsAsBefore(ResultCodes code) throws IOException { + failFsoRename(code); + OMException fromGetBucket = new OMException("missing", code); + when(adapter.getBucket(any(OFSPath.class), anyBoolean())).thenThrow(fromGetBucket); + + assertSame(fromGetBucket, assertThrows(OMException.class, () -> fs.rename(SRC, DST))); + } + + @Test + public void olderOmFetchesTheBucket() throws IOException { + when(adapter.isRenameRequiringFileSystemOptimizedSupported()).thenReturn(false); + when(bucket.getBucketLayout()).thenReturn(BucketLayout.FILE_SYSTEM_OPTIMIZED); + + assertTrue(fs.rename(SRC, DST)); + verify(adapter).getBucket(any(OFSPath.class), anyBoolean()); + verify(adapter).rename(bucket, SRC_KEY_PATH, DST_KEY_PATH); + verify(adapter, never()).renameRequiringFileSystemOptimized(anyString(), anyString()); + } + + /** The adapter sends the rename with the field set and gates it on the OM version. */ + @Test + public void adapterSendsTheFlaggedRenameOnlyToANewOm() throws Exception { + BasicRootedOzoneClientAdapterImpl realAdapter = mock(BasicRootedOzoneClientAdapterImpl.class, CALLS_REAL_METHODS); + ClientProtocol proxy = mock(ClientProtocol.class); + setField(realAdapter, "proxy", proxy); + setField(realAdapter, "config", new OzoneConfiguration()); + + realAdapter.renameRequiringFileSystemOptimized(SRC_KEY_PATH, DST_KEY_PATH); + verify(proxy).renameKey("vol", "bucket", "dir/file", "dir/file2", true); + + when(proxy.getOmVersion()).thenReturn(OzoneManagerVersion.RENAME_KEY_REJECTS_NON_FSO); + assertTrue(realAdapter.isRenameRequiringFileSystemOptimizedSupported()); + // An OM finalized for ZDU, or at a later version without this change, must not be trusted to reject. + for (OzoneManagerVersion older : new OzoneManagerVersion[] { + OzoneManagerVersion.ZDU, OzoneManagerVersion.GET_KEY_INFO_REJECTS_OBS}) { + when(proxy.getOmVersion()).thenReturn(older); + assertFalse(realAdapter.isRenameRequiringFileSystemOptimizedSupported(), older.name()); + } + } + + private static void setField(BasicRootedOzoneClientAdapterImpl target, String name, Object value) throws Exception { + Field field = BasicRootedOzoneClientAdapterImpl.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index d5d75b27770f..c95e49c481e3 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -419,6 +419,11 @@ public void renameKey(String volumeName, String bucketName, } + @Override + public void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName, + boolean requireFileSystemOptimized) throws IOException { + } + @Override public void renameKeys(String volumeName, String bucketName, Map keyMap) throws IOException {