HDDS-16951. Remove the per-call InfoBucket from rooted-ofs rename on FSO buckets

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -97,6 +97,12 @@ public enum OzoneManagerVersion implements ComponentVersion {
       "OzoneManager version that gives each multipart upload part in an "
           + "encrypted bucket its own IV"),
 
+  RENAME_KEY_REJECTS_NON_FSO(106,
+      "OzoneManager version that rejects renameKey on buckets that are not "
+          + "FILE_SYSTEM_OPTIMIZED when the request sets requireFileSystemOptimized, "
+          + "so file system clients no longer need the client-side InfoBucket "
+          + "layout check for rename"),
+
   UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
       " component, which may indicate it came from a component in a newer version");
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3ClientProtocol.java

@@ -394,6 +394,12 @@ public void renameKey(String volumeName, String bucketName, String fromKeyName,
     delegate.renameKey(volumeName, bucketName, fromKeyName, toKeyName);
   }
 
+  @Override
+  public void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName,
+      boolean requireFileSystemOptimized) throws IOException {
+    delegate.renameKey(volumeName, bucketName, fromKeyName, toKeyName, requireFileSystemOptimized);
+  }
+
   @Override
   @Deprecated
   public void renameKeys(String volumeName, String bucketName, Map<String, String> keyMap) throws IOException {

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -713,6 +713,18 @@ Map<String, ErrorInfo> deleteKeys(String volumeName, String bucketName,
   void renameKey(String volumeName, String bucketName, String fromKeyName,
                  String toKeyName) throws IOException;
 
+  /**
+   * Renames an existing key within a bucket, like
+   * {@link #renameKey(String, String, String, String)}.
+   *
+   * @param requireFileSystemOptimized if true, OM rejects the rename with
+   *     NOT_SUPPORTED_OPERATION unless the resolved bucket layout is
+   *     FILE_SYSTEM_OPTIMIZED. An OM older than
+   *     OzoneManagerVersion.RENAME_KEY_REJECTS_NON_FSO ignores it.
+   */
+  void renameKey(String volumeName, String bucketName, String fromKeyName,
+      String toKeyName, boolean requireFileSystemOptimized) throws IOException;
+
   /**
    * Renames existing keys within a bucket.
    * @param volumeName Name of the Volume

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -1870,6 +1870,12 @@ public Map<String, ErrorInfo> deleteKeys(
   @Override
   public void renameKey(String volumeName, String bucketName,
       String fromKeyName, String toKeyName) throws IOException {
+    renameKey(volumeName, bucketName, fromKeyName, toKeyName, false);
+  }
+
+  @Override
+  public void renameKey(String volumeName, String bucketName,
+      String fromKeyName, String toKeyName, boolean requireFileSystemOptimized) throws IOException {
     verifyVolumeName(volumeName);
     verifyBucketName(bucketName);
     if (checkKeyNameEnabled) {
@@ -1880,6 +1886,7 @@ public void renameKey(String volumeName, String bucketName,
         .setVolumeName(volumeName)
         .setBucketName(bucketName)
         .setKeyName(fromKeyName)
+        .setRequireFileSystemOptimized(requireFileSystemOptimized)
         .build();
     ozoneManagerClient.renameKey(keyArgs, toKeyName);
   }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java

@@ -67,6 +67,9 @@ public final class OmKeyArgs extends WithMetadata implements Auditable {
   // Set by file system clients so that OM rejects a listing or a key info
   // read on an OBJECT_STORE bucket.
   private final boolean requireFileSystemLayout;
+  // Set by file system clients so that OM rejects a rename unless the bucket
+  // is FILE_SYSTEM_OPTIMIZED.
+  private final boolean requireFileSystemOptimized;
   private final String expectedETag;
   private final String excludedETag;
   private final boolean derivedKeyPiggyBacking;
@@ -93,6 +96,7 @@ private OmKeyArgs(Builder b) {
     this.expectedDataGeneration = b.expectedDataGeneration;
     this.listPrefix = b.listPrefix;
     this.requireFileSystemLayout = b.requireFileSystemLayout;
+    this.requireFileSystemOptimized = b.requireFileSystemOptimized;
     this.expectedETag = b.expectedETag;
     this.excludedETag = b.excludedETag;
     this.derivedKeyPiggyBacking = b.derivedKeyPiggyBacking;
@@ -194,6 +198,14 @@ public boolean isRequireFileSystemLayout() {
     return requireFileSystemLayout;
   }
 
+  /**
+   * Whether OM must reject a rename unless the resolved bucket layout is
+   * FILE_SYSTEM_OPTIMIZED, because the caller relies on atomic directory rename.
+   */
+  public boolean isRequireFileSystemOptimized() {
+    return requireFileSystemOptimized;
+  }
+
   public String getExpectedETag() {
     return expectedETag;
   }
@@ -284,6 +296,7 @@ public static class Builder extends WithMetadata.Builder {
     private Long expectedDataGeneration = null;
     private String listPrefix = null;
     private boolean requireFileSystemLayout;
+    private boolean requireFileSystemOptimized;
     private String expectedETag;
     private String excludedETag;
     private boolean derivedKeyPiggyBacking;
@@ -339,6 +352,7 @@ public Builder(OmKeyArgs obj) {
       this.acls = AclListBuilder.of(obj.acls);
       this.listPrefix = obj.listPrefix;
       this.requireFileSystemLayout = obj.requireFileSystemLayout;
+      this.requireFileSystemOptimized = obj.requireFileSystemOptimized;
     }
 
     public Builder setVolumeName(String volume) {
@@ -482,6 +496,11 @@ public Builder setRequireFileSystemLayout(boolean require) {
       return this;
     }
 
+    public Builder setRequireFileSystemOptimized(boolean require) {
+      this.requireFileSystemOptimized = require;
+      return this;
+    }
+
     public Builder setExpectedETag(String eTag) {
       this.expectedETag = eTag;
       return this;

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -998,6 +998,9 @@ public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
         .setDataSize(args.getDataSize()).build();
     req.setKeyArgs(keyArgs);
     req.setToKeyName(toKeyName);
+    if (args.isRequireFileSystemOptimized()) {
+      req.setRequireFileSystemOptimized(true);
+    }
 
     OMRequest omRequest = createOMRequest(Type.RenameKey)
         .setRenameKeyRequest(req)

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java

@@ -2029,6 +2029,45 @@ void testOpenInMissingBucketFails() throws Exception {
     assertEquals(BUCKET_NOT_FOUND, exception.getResult());
   }
 
+  @Test
+  void testRenameDoesNotTriggerInfoBucketOnFso() throws Exception {
+    assumeTrue(bucketLayout.isFileSystemOptimized(), "Only a rename in an FSO bucket skips the bucket lookup");
+    Path dir = new Path(bucketPath, "rename-rpc-" + RandomStringUtils.secure().nextAlphanumeric(5));
+    Path src = new Path(dir, "file");
+    Path dst = new Path(dir, "renamed");
+    ContractTestUtils.touch(fs, src);
+
+    OMMetrics metrics = getOMMetrics();
+    long bucketInfosBefore = metrics.getNumBucketInfos();
+
+    assertTrue(fs.rename(src, dst));
+
+    assertEquals(bucketInfosBefore, metrics.getNumBucketInfos(), "an FSO rename must not trigger InfoBucket");
+    assertTrue(fs.exists(dst));
+    assertFalse(fs.exists(src));
+  }
+
+  @Test
+  void testRenameRejectsObsBucket() throws Exception {
+    // OM rejects the rename, and the client then rejects the bucket itself, as before.
+    OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE);
+    DataTestUtil.createKey(obsBucket, "obs-key", "data".getBytes(StandardCharsets.UTF_8));
+    Path obsBucketPath = new Path(new Path(OZONE_URI_DELIMITER, obsBucket.getVolumeName()), obsBucket.getName());
+
+    IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
+        () -> fs.rename(new Path(obsBucketPath, "obs-key"), new Path(obsBucketPath, "obs-key-2")));
+    assertThat(exception.getMessage()).contains(obsBucket.getName());
+    assertThat(exception.getMessage()).contains("OBJECT_STORE");
+  }
+
+  @Test
+  void testRenameInMissingBucketFails() throws Exception {
+    Path missingBucket = new Path(new Path(OZONE_URI_DELIMITER, volumeName), "missing-bucket");
+    OMException exception = assertThrows(OMException.class,
+        () -> fs.rename(new Path(missingBucket, "file"), new Path(missingBucket, "file-2")));
+    assertEquals(BUCKET_NOT_FOUND, exception.getResult());
+  }
+
   @Test
   void testGetFileStatus() throws Exception {
     String volumeNameLocal = getRandomNonExistVolumeName();