HDDS-17502. Report purgeable bytes split by deleted keys and directories in GetBucketDeletedBytes

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java

@@ -36,20 +36,22 @@ public final class BucketDeletedBytes {
   private final long purgeableKeys;
   private final long snapshotTrappedDirs;
   private final long purgeableDirs;
+  private final long purgeableKeyBytes;
+  private final long purgeableDirBytes;
 
-  public BucketDeletedBytes(
-      long snapshotTrappedBytes,
-      long purgeableBytes,
-      long snapshotTrappedKeys,
-      long purgeableKeys,
-      long snapshotTrappedDirs,
-      long purgeableDirs) {
-    this.snapshotTrappedBytes = snapshotTrappedBytes;
-    this.purgeableBytes = purgeableBytes;
-    this.snapshotTrappedKeys = snapshotTrappedKeys;
-    this.purgeableKeys = purgeableKeys;
-    this.snapshotTrappedDirs = snapshotTrappedDirs;
-    this.purgeableDirs = purgeableDirs;
+  private BucketDeletedBytes(Builder b) {
+    this.snapshotTrappedBytes = b.snapshotTrappedBytes;
+    this.purgeableBytes = b.purgeableBytes;
+    this.snapshotTrappedKeys = b.snapshotTrappedKeys;
+    this.purgeableKeys = b.purgeableKeys;
+    this.snapshotTrappedDirs = b.snapshotTrappedDirs;
+    this.purgeableDirs = b.purgeableDirs;
+    this.purgeableKeyBytes = b.purgeableKeyBytes;
+    this.purgeableDirBytes = b.purgeableDirBytes;
+  }
+
+  public static Builder newBuilder() {
+    return new Builder();
   }
 
   public long getSnapshotTrappedBytes() {
@@ -75,5 +77,79 @@ public long getSnapshotTrappedDirs() {
   public long getPurgeableDirs() {
     return purgeableDirs;
   }
-}
 
+  /**
+   * Purgeable bytes of entries in the deleted table, or -1 if OM did not report it.
+   */
+  public long getPurgeableKeyBytes() {
+    return purgeableKeyBytes;
+  }
+
+  /**
+   * Purgeable bytes of files found under deleted directories, or -1 if OM did not report it.
+   */
+  public long getPurgeableDirBytes() {
+    return purgeableDirBytes;
+  }
+
+  /**
+   * Builder for BucketDeletedBytes.
+   */
+  public static final class Builder {
+    private long snapshotTrappedBytes;
+    private long purgeableBytes;
+    private long snapshotTrappedKeys;
+    private long purgeableKeys;
+    private long snapshotTrappedDirs;
+    private long purgeableDirs;
+    private long purgeableKeyBytes = -1L;
+    private long purgeableDirBytes = -1L;
+
+    private Builder() {
+    }
+
+    public Builder setSnapshotTrappedBytes(long snapshotTrappedBytes) {
+      this.snapshotTrappedBytes = snapshotTrappedBytes;
+      return this;
+    }
+
+    public Builder setPurgeableBytes(long purgeableBytes) {
+      this.purgeableBytes = purgeableBytes;
+      return this;
+    }
+
+    public Builder setSnapshotTrappedKeys(long snapshotTrappedKeys) {
+      this.snapshotTrappedKeys = snapshotTrappedKeys;
+      return this;
+    }
+
+    public Builder setPurgeableKeys(long purgeableKeys) {
+      this.purgeableKeys = purgeableKeys;
+      return this;
+    }
+
+    public Builder setSnapshotTrappedDirs(long snapshotTrappedDirs) {
+      this.snapshotTrappedDirs = snapshotTrappedDirs;
+      return this;
+    }
+
+    public Builder setPurgeableDirs(long purgeableDirs) {
+      this.purgeableDirs = purgeableDirs;
+      return this;
+    }
+
+    public Builder setPurgeableKeyBytes(long purgeableKeyBytes) {
+      this.purgeableKeyBytes = purgeableKeyBytes;
+      return this;
+    }
+
+    public Builder setPurgeableDirBytes(long purgeableDirBytes) {
+      this.purgeableDirBytes = purgeableDirBytes;
+      return this;
+    }
+
+    public BucketDeletedBytes build() {
+      return new BucketDeletedBytes(this);
+    }
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -2110,13 +2110,20 @@ public BucketDeletedBytes getBucketDeletedBytes(String bucketPath)
 
     GetBucketDeletedBytesResponse response = handleError(submitRequest(omRequest))
         .getGetBucketDeletedBytesResponse();
-    return new BucketDeletedBytes(
-        response.getSnapshotTrappedBytes(),
-        response.getPurgeableBytes(),
-        response.getSnapshotTrappedKeys(),
-        response.getPurgeableKeys(),
-        response.getSnapshotTrappedDirs(),
-        response.getPurgeableDirs());
+    BucketDeletedBytes.Builder builder = BucketDeletedBytes.newBuilder()
+        .setSnapshotTrappedBytes(response.getSnapshotTrappedBytes())
+        .setPurgeableBytes(response.getPurgeableBytes())
+        .setSnapshotTrappedKeys(response.getSnapshotTrappedKeys())
+        .setPurgeableKeys(response.getPurgeableKeys())
+        .setSnapshotTrappedDirs(response.getSnapshotTrappedDirs())
+        .setPurgeableDirs(response.getPurgeableDirs());
+    if (response.hasPurgeableKeyBytes()) {
+      builder.setPurgeableKeyBytes(response.getPurgeableKeyBytes());
+    }
+    if (response.hasPurgeableDirBytes()) {
+      builder.setPurgeableDirBytes(response.getPurgeableDirBytes());
+    }
+    return builder.build();
   }
 
   @Override

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestOzoneManagerProtocolClientSideTranslatorPB.java

@@ -34,13 +34,15 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes;
 import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
 import org.apache.hadoop.ozone.om.helpers.OmDeleteKeys;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.ReadConsistency;
 import org.apache.hadoop.ozone.om.protocol.S3Auth;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketDeletedBytesResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
@@ -261,6 +263,36 @@ void testQuietDeleteKeysReturnsEmptyMapOnSuccess() throws IOException {
     assertThat(pb.deleteKeys(DELETE_KEYS, true)).isEmpty();
   }
 
+  @Test
+  void testGetBucketDeletedBytesFromOlderOmWithoutPurgeableBytesSplit() throws IOException {
+    // Older OM does not set purgeableKeyBytes / purgeableDirBytes in the response.
+    GetBucketDeletedBytesResponse response = GetBucketDeletedBytesResponse.newBuilder()
+        .setSnapshotTrappedBytes(100L)
+        .setPurgeableBytes(200L)
+        .setSnapshotTrappedKeys(1L)
+        .setPurgeableKeys(2L)
+        .setSnapshotTrappedDirs(3L)
+        .setPurgeableDirs(4L)
+        .build();
+    when(omTransport.submitRequest(any(OMRequest.class))).thenReturn(
+        OMResponse.newBuilder()
+            .setCmdType(Type.GetBucketDeletedBytes)
+            .setStatus(Status.OK)
+            .setGetBucketDeletedBytesResponse(response)
+            .build());
+
+    BucketDeletedBytes bytes = pb.getBucketDeletedBytes("/vol/bucket");
+
+    assertThat(bytes.getSnapshotTrappedBytes()).isEqualTo(100L);
+    assertThat(bytes.getPurgeableBytes()).isEqualTo(200L);
+    assertThat(bytes.getSnapshotTrappedKeys()).isEqualTo(1L);
+    assertThat(bytes.getPurgeableKeys()).isEqualTo(2L);
+    assertThat(bytes.getSnapshotTrappedDirs()).isEqualTo(3L);
+    assertThat(bytes.getPurgeableDirs()).isEqualTo(4L);
+    assertThat(bytes.getPurgeableKeyBytes()).isEqualTo(-1L);
+    assertThat(bytes.getPurgeableDirBytes()).isEqualTo(-1L);
+  }
+
   private static final class CapturingTransport implements OmTransport {
     private OMRequest lastRequest;
 

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -1855,6 +1855,8 @@ message GetBucketDeletedBytesResponse {
   optional uint64 purgeableKeys = 4;
   optional uint64 snapshotTrappedDirs = 5;
   optional uint64 purgeableDirs = 6;
+  optional uint64 purgeableKeyBytes = 7;
+  optional uint64 purgeableDirBytes = 8;
 }
 
 message ServicePort {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -1959,13 +1959,16 @@ public BucketDeletedBytes getBucketDeletedBytes(String bucketPath)
     checkAdminUserPrivilege("get bucket deleted bytes.");
     BucketDeletedDataCalculator.BucketDeletedBytesStats stats =
         calculateDeletedBytesForBucket(bucketPath);
-    return new BucketDeletedBytes(
-        stats.getSnapshotTrappedBytes(),
-        stats.getPurgeableBytes(),
-        stats.getSnapshotTrappedKeys(),
-        stats.getPurgeableKeys(),
-        stats.getSnapshotTrappedDirs(),
-        stats.getPurgeableDirs());
+    return BucketDeletedBytes.newBuilder()
+        .setSnapshotTrappedBytes(stats.getSnapshotTrappedBytes())
+        .setPurgeableBytes(stats.getPurgeableBytes())
+        .setSnapshotTrappedKeys(stats.getSnapshotTrappedKeys())
+        .setPurgeableKeys(stats.getPurgeableKeys())
+        .setSnapshotTrappedDirs(stats.getSnapshotTrappedDirs())
+        .setPurgeableDirs(stats.getPurgeableDirs())
+        .setPurgeableKeyBytes(stats.getPurgeableKeyBytes())
+        .setPurgeableDirBytes(stats.getPurgeableDirBytes())
+        .build();
   }
 
   /**

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java

@@ -259,7 +259,7 @@ private void traverseDeletedDirSubtree(
         boolean fileReclaimable = currentDirReclaimable
             || reclaimableKeyFilter.apply(Table.newKeyValue("", subFile));
         if (fileReclaimable) {
-          totals.addPurgeableKey(subFile.getReplicatedSize());
+          totals.addPurgeableDirFile(subFile.getReplicatedSize());
         } else {
           totals.addSnapshotTrappedKey(subFile.getReplicatedSize());
         }
@@ -318,6 +318,8 @@ public static final class BucketDeletedBytesStats {
     private long purgeableKeys;
     private long snapshotTrappedDirs;
     private long purgeableDirs;
+    private long purgeableKeyBytes;
+    private long purgeableDirBytes;
 
     public long getSnapshotTrappedBytes() {
       return snapshotTrappedBytes;
@@ -343,6 +345,16 @@ public long getPurgeableDirs() {
       return purgeableDirs;
     }
 
+    /** Purgeable bytes of entries in the deleted table. */
+    public long getPurgeableKeyBytes() {
+      return purgeableKeyBytes;
+    }
+
+    /** Purgeable bytes of files found under deleted directories. */
+    public long getPurgeableDirBytes() {
+      return purgeableDirBytes;
+    }
+
     public void addSnapshotTrappedBytes(long value) {
       snapshotTrappedBytes += value;
     }
@@ -358,6 +370,13 @@ public void addSnapshotTrappedKey(long bytes) {
 
     public void addPurgeableKey(long bytes) {
       addPurgeableBytes(bytes);
+      purgeableKeyBytes += bytes;
+      purgeableKeys++;
+    }
+
+    public void addPurgeableDirFile(long bytes) {
+      addPurgeableBytes(bytes);
+      purgeableDirBytes += bytes;
       purgeableKeys++;
     }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java

@@ -1029,6 +1029,8 @@ private GetBucketDeletedBytesResponse getBucketDeletedBytes(
         .setPurgeableKeys(bytes.getPurgeableKeys())
         .setSnapshotTrappedDirs(bytes.getSnapshotTrappedDirs())
         .setPurgeableDirs(bytes.getPurgeableDirs())
+        .setPurgeableKeyBytes(bytes.getPurgeableKeyBytes())
+        .setPurgeableDirBytes(bytes.getPurgeableDirBytes())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java

@@ -135,6 +135,8 @@ public void testAosDeletedKeyReportedAsPurgeable() throws Exception {
     assertTrue(stats.getPurgeableBytes() > 0L);
     assertEquals(1L, stats.getPurgeableKeys());
     assertEquals(0L, stats.getPurgeableDirs());
+    assertEquals(stats.getPurgeableBytes(), stats.getPurgeableKeyBytes());
+    assertEquals(0L, stats.getPurgeableDirBytes());
   }
 
   @Test
@@ -238,6 +240,8 @@ public void testAosDeletedDirTraversalReportedAsPurgeable() throws Exception {
     assertEquals(1L, stats.getPurgeableKeys());
     assertEquals(0L, stats.getSnapshotTrappedDirs());
     assertEquals(1L, stats.getPurgeableDirs());
+    assertEquals(0L, stats.getPurgeableKeyBytes());
+    assertEquals(subFileInfo.getReplicatedSize(), stats.getPurgeableDirBytes());
   }
 }