diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java index 36967b25eca6..3fcc4d6b692c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java @@ -36,20 +36,22 @@ public final class BucketDeletedBytes { private final long purgeableKeys; private final long snapshotTrappedDirs; private final long purgeableDirs; + private final long purgeableKeyBytes; + private final long purgeableDirBytes; - public BucketDeletedBytes( - long snapshotTrappedBytes, - long purgeableBytes, - long snapshotTrappedKeys, - long purgeableKeys, - long snapshotTrappedDirs, - long purgeableDirs) { - this.snapshotTrappedBytes = snapshotTrappedBytes; - this.purgeableBytes = purgeableBytes; - this.snapshotTrappedKeys = snapshotTrappedKeys; - this.purgeableKeys = purgeableKeys; - this.snapshotTrappedDirs = snapshotTrappedDirs; - this.purgeableDirs = purgeableDirs; + private BucketDeletedBytes(Builder b) { + this.snapshotTrappedBytes = b.snapshotTrappedBytes; + this.purgeableBytes = b.purgeableBytes; + this.snapshotTrappedKeys = b.snapshotTrappedKeys; + this.purgeableKeys = b.purgeableKeys; + this.snapshotTrappedDirs = b.snapshotTrappedDirs; + this.purgeableDirs = b.purgeableDirs; + this.purgeableKeyBytes = b.purgeableKeyBytes; + this.purgeableDirBytes = b.purgeableDirBytes; + } + + public static Builder newBuilder() { + return new Builder(); } public long getSnapshotTrappedBytes() { @@ -75,5 +77,79 @@ public long getSnapshotTrappedDirs() { public long getPurgeableDirs() { return purgeableDirs; } -} + /** + * Purgeable bytes of entries in the deleted table, or -1 if OM did not report it. + */ + public long getPurgeableKeyBytes() { + return purgeableKeyBytes; + } + + /** + * Purgeable bytes of files found under deleted directories, or -1 if OM did not report it. + */ + public long getPurgeableDirBytes() { + return purgeableDirBytes; + } + + /** + * Builder for BucketDeletedBytes. + */ + public static final class Builder { + private long snapshotTrappedBytes; + private long purgeableBytes; + private long snapshotTrappedKeys; + private long purgeableKeys; + private long snapshotTrappedDirs; + private long purgeableDirs; + private long purgeableKeyBytes = -1L; + private long purgeableDirBytes = -1L; + + private Builder() { + } + + public Builder setSnapshotTrappedBytes(long snapshotTrappedBytes) { + this.snapshotTrappedBytes = snapshotTrappedBytes; + return this; + } + + public Builder setPurgeableBytes(long purgeableBytes) { + this.purgeableBytes = purgeableBytes; + return this; + } + + public Builder setSnapshotTrappedKeys(long snapshotTrappedKeys) { + this.snapshotTrappedKeys = snapshotTrappedKeys; + return this; + } + + public Builder setPurgeableKeys(long purgeableKeys) { + this.purgeableKeys = purgeableKeys; + return this; + } + + public Builder setSnapshotTrappedDirs(long snapshotTrappedDirs) { + this.snapshotTrappedDirs = snapshotTrappedDirs; + return this; + } + + public Builder setPurgeableDirs(long purgeableDirs) { + this.purgeableDirs = purgeableDirs; + return this; + } + + public Builder setPurgeableKeyBytes(long purgeableKeyBytes) { + this.purgeableKeyBytes = purgeableKeyBytes; + return this; + } + + public Builder setPurgeableDirBytes(long purgeableDirBytes) { + this.purgeableDirBytes = purgeableDirBytes; + return this; + } + + public BucketDeletedBytes build() { + return new BucketDeletedBytes(this); + } + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index baa22a94e783..49bae380f6c2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2110,13 +2110,20 @@ public BucketDeletedBytes getBucketDeletedBytes(String bucketPath) GetBucketDeletedBytesResponse response = handleError(submitRequest(omRequest)) .getGetBucketDeletedBytesResponse(); - return new BucketDeletedBytes( - response.getSnapshotTrappedBytes(), - response.getPurgeableBytes(), - response.getSnapshotTrappedKeys(), - response.getPurgeableKeys(), - response.getSnapshotTrappedDirs(), - response.getPurgeableDirs()); + BucketDeletedBytes.Builder builder = BucketDeletedBytes.newBuilder() + .setSnapshotTrappedBytes(response.getSnapshotTrappedBytes()) + .setPurgeableBytes(response.getPurgeableBytes()) + .setSnapshotTrappedKeys(response.getSnapshotTrappedKeys()) + .setPurgeableKeys(response.getPurgeableKeys()) + .setSnapshotTrappedDirs(response.getSnapshotTrappedDirs()) + .setPurgeableDirs(response.getPurgeableDirs()); + if (response.hasPurgeableKeyBytes()) { + builder.setPurgeableKeyBytes(response.getPurgeableKeyBytes()); + } + if (response.hasPurgeableDirBytes()) { + builder.setPurgeableDirBytes(response.getPurgeableDirBytes()); + } + return builder.build(); } @Override diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestOzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestOzoneManagerProtocolClientSideTranslatorPB.java index d899b53a918b..cda863cc67db 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestOzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestOzoneManagerProtocolClientSideTranslatorPB.java @@ -34,6 +34,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmDeleteKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -41,6 +42,7 @@ import org.apache.hadoop.ozone.om.protocol.S3Auth; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketDeletedBytesResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse; @@ -261,6 +263,36 @@ void testQuietDeleteKeysReturnsEmptyMapOnSuccess() throws IOException { assertThat(pb.deleteKeys(DELETE_KEYS, true)).isEmpty(); } + @Test + void testGetBucketDeletedBytesFromOlderOmWithoutPurgeableBytesSplit() throws IOException { + // Older OM does not set purgeableKeyBytes / purgeableDirBytes in the response. + GetBucketDeletedBytesResponse response = GetBucketDeletedBytesResponse.newBuilder() + .setSnapshotTrappedBytes(100L) + .setPurgeableBytes(200L) + .setSnapshotTrappedKeys(1L) + .setPurgeableKeys(2L) + .setSnapshotTrappedDirs(3L) + .setPurgeableDirs(4L) + .build(); + when(omTransport.submitRequest(any(OMRequest.class))).thenReturn( + OMResponse.newBuilder() + .setCmdType(Type.GetBucketDeletedBytes) + .setStatus(Status.OK) + .setGetBucketDeletedBytesResponse(response) + .build()); + + BucketDeletedBytes bytes = pb.getBucketDeletedBytes("/vol/bucket"); + + assertThat(bytes.getSnapshotTrappedBytes()).isEqualTo(100L); + assertThat(bytes.getPurgeableBytes()).isEqualTo(200L); + assertThat(bytes.getSnapshotTrappedKeys()).isEqualTo(1L); + assertThat(bytes.getPurgeableKeys()).isEqualTo(2L); + assertThat(bytes.getSnapshotTrappedDirs()).isEqualTo(3L); + assertThat(bytes.getPurgeableDirs()).isEqualTo(4L); + assertThat(bytes.getPurgeableKeyBytes()).isEqualTo(-1L); + assertThat(bytes.getPurgeableDirBytes()).isEqualTo(-1L); + } + private static final class CapturingTransport implements OmTransport { private OMRequest lastRequest; diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 883977253973..c60df10c2589 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1855,6 +1855,8 @@ message GetBucketDeletedBytesResponse { optional uint64 purgeableKeys = 4; optional uint64 snapshotTrappedDirs = 5; optional uint64 purgeableDirs = 6; + optional uint64 purgeableKeyBytes = 7; + optional uint64 purgeableDirBytes = 8; } message ServicePort { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index dc0bc020c827..950cbbae0143 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1959,13 +1959,16 @@ public BucketDeletedBytes getBucketDeletedBytes(String bucketPath) checkAdminUserPrivilege("get bucket deleted bytes."); BucketDeletedDataCalculator.BucketDeletedBytesStats stats = calculateDeletedBytesForBucket(bucketPath); - return new BucketDeletedBytes( - stats.getSnapshotTrappedBytes(), - stats.getPurgeableBytes(), - stats.getSnapshotTrappedKeys(), - stats.getPurgeableKeys(), - stats.getSnapshotTrappedDirs(), - stats.getPurgeableDirs()); + return BucketDeletedBytes.newBuilder() + .setSnapshotTrappedBytes(stats.getSnapshotTrappedBytes()) + .setPurgeableBytes(stats.getPurgeableBytes()) + .setSnapshotTrappedKeys(stats.getSnapshotTrappedKeys()) + .setPurgeableKeys(stats.getPurgeableKeys()) + .setSnapshotTrappedDirs(stats.getSnapshotTrappedDirs()) + .setPurgeableDirs(stats.getPurgeableDirs()) + .setPurgeableKeyBytes(stats.getPurgeableKeyBytes()) + .setPurgeableDirBytes(stats.getPurgeableDirBytes()) + .build(); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java index 5fc2b144bee2..95105c2df50f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java @@ -259,7 +259,7 @@ private void traverseDeletedDirSubtree( boolean fileReclaimable = currentDirReclaimable || reclaimableKeyFilter.apply(Table.newKeyValue("", subFile)); if (fileReclaimable) { - totals.addPurgeableKey(subFile.getReplicatedSize()); + totals.addPurgeableDirFile(subFile.getReplicatedSize()); } else { totals.addSnapshotTrappedKey(subFile.getReplicatedSize()); } @@ -318,6 +318,8 @@ public static final class BucketDeletedBytesStats { private long purgeableKeys; private long snapshotTrappedDirs; private long purgeableDirs; + private long purgeableKeyBytes; + private long purgeableDirBytes; public long getSnapshotTrappedBytes() { return snapshotTrappedBytes; @@ -343,6 +345,16 @@ public long getPurgeableDirs() { return purgeableDirs; } + /** Purgeable bytes of entries in the deleted table. */ + public long getPurgeableKeyBytes() { + return purgeableKeyBytes; + } + + /** Purgeable bytes of files found under deleted directories. */ + public long getPurgeableDirBytes() { + return purgeableDirBytes; + } + public void addSnapshotTrappedBytes(long value) { snapshotTrappedBytes += value; } @@ -358,6 +370,13 @@ public void addSnapshotTrappedKey(long bytes) { public void addPurgeableKey(long bytes) { addPurgeableBytes(bytes); + purgeableKeyBytes += bytes; + purgeableKeys++; + } + + public void addPurgeableDirFile(long bytes) { + addPurgeableBytes(bytes); + purgeableDirBytes += bytes; purgeableKeys++; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 5d6be583ab20..a85ebaf975db 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1029,6 +1029,8 @@ private GetBucketDeletedBytesResponse getBucketDeletedBytes( .setPurgeableKeys(bytes.getPurgeableKeys()) .setSnapshotTrappedDirs(bytes.getSnapshotTrappedDirs()) .setPurgeableDirs(bytes.getPurgeableDirs()) + .setPurgeableKeyBytes(bytes.getPurgeableKeyBytes()) + .setPurgeableDirBytes(bytes.getPurgeableDirBytes()) .build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java index a3b8e1334b73..fe20fab20643 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java @@ -135,6 +135,8 @@ public void testAosDeletedKeyReportedAsPurgeable() throws Exception { assertTrue(stats.getPurgeableBytes() > 0L); assertEquals(1L, stats.getPurgeableKeys()); assertEquals(0L, stats.getPurgeableDirs()); + assertEquals(stats.getPurgeableBytes(), stats.getPurgeableKeyBytes()); + assertEquals(0L, stats.getPurgeableDirBytes()); } @Test @@ -238,6 +240,8 @@ public void testAosDeletedDirTraversalReportedAsPurgeable() throws Exception { assertEquals(1L, stats.getPurgeableKeys()); assertEquals(0L, stats.getSnapshotTrappedDirs()); assertEquals(1L, stats.getPurgeableDirs()); + assertEquals(0L, stats.getPurgeableKeyBytes()); + assertEquals(subFileInfo.getReplicatedSize(), stats.getPurgeableDirBytes()); } }