Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

/** The main class of multiple directories. Used to allocate folders to data files. */
Expand All @@ -62,7 +64,10 @@

private record TierDirectory(int tierLevel, String folderName) {}

private volatile Map<TierDirectory, TierFolderManager> folderManagers = Map.of();
/** Replaced as a whole by {@link #rebuildFolders()}; the published maps are unmodifiable. */

Check warning on line 67 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Single-line Javadoc comment should be multi-line.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHKnF546Gx163_JXpj3&open=AaHKnF546Gx163_JXpj3&pullRequest=19228
private final AtomicReference<Map<TierDirectory, TierFolderManager>> folderManagers =
new AtomicReference<>(Map.of());

// A failed constructor can report DiskFull before the replacement managers are published.
private volatile boolean rebuildingFolders;

Expand All @@ -75,7 +80,7 @@
private List<String> objectDirs;

/** total space of each tier, Long.MAX_VALUE when one tier contains remote storage */
private volatile long[] tierDiskTotalSpace = new long[0];
private final AtomicReference<long[]> tierDiskTotalSpace = new AtomicReference<>(new long[0]);

private TierManager() {
initFolders();
Expand All @@ -91,7 +96,7 @@
}
}

private void rebuildFolders() {

Check failure on line 99 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 27 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHKnF546Gx163_JXpj1&open=AaHKnF546Gx163_JXpj1&pullRequest=19228
Map<TierDirectory, TierFolderManager> newManagers = new HashMap<>();
Map<String, Integer> newSeqDir2TierLevel = new HashMap<>();
Map<String, Integer> newUnSeqDir2TierLevel = new HashMap<>();
Expand Down Expand Up @@ -206,10 +211,10 @@
}
}

tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL);
tierDiskTotalSpace.set(getTierDiskSpace(DiskSpaceType.TOTAL));
seqDir2TierLevel = newSeqDir2TierLevel;
unSeqDir2TierLevel = newUnSeqDir2TierLevel;
folderManagers = newManagers;
folderManagers.set(Collections.unmodifiableMap(newManagers));
}

private void addFolderManager(
Expand All @@ -233,28 +238,30 @@
private final class TierFolderManager {
private final List<String> folders;
private final MemoizedCheckedSupplier<FolderManager, DiskSpaceInsufficientException> supplier;
private volatile FolderManager manager;
private final AtomicReference<FolderManager> manager = new AtomicReference<>();

private TierFolderManager(TierDirectory key, List<String> folders, DirectoryStrategyType type) {
this.folders = folders;
supplier =
MemoizedCheckedSupplier.valueOf(
() -> {
manager = new FolderManager(folders, type, () -> reportDiskFull(key, this));
return manager;
FolderManager created =
new FolderManager(folders, type, () -> reportDiskFull(key, this));
manager.set(created);
return created;
});
}

private boolean hasSpaceForRecovery() {
FolderManager current = manager;
FolderManager current = manager.get();
return current == null
? folders.stream().anyMatch(JVMCommonUtils::hasSpaceWithoutRetry)
: current.hasSpaceForRecovery();
}
}

private void reportDiskFull(TierDirectory key, TierFolderManager source) {

Check warning on line 263 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Move this method into "TierFolderManager".

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHKnF546Gx163_JXpj2&open=AaHKnF546Gx163_JXpj2&pullRequest=19228
if (!rebuildingFolders && folderManagers.get(key) != source) {
if (!rebuildingFolders && folderManagers.get().get(key) != source) {
return;
}
CommonDescriptor.getInstance()
Expand All @@ -263,13 +270,13 @@
}

private boolean hasSpaceForRecovery(TierDirectory key) {
Map<TierDirectory, TierFolderManager> active = folderManagers;
Map<TierDirectory, TierFolderManager> active = folderManagers.get();
if (rebuildingFolders) {
return false;
}
TierFolderManager current = active.get(key);
boolean recovered = current == null || current.hasSpaceForRecovery();
return recovered && active == folderManagers && !rebuildingFolders;
return recovered && active == folderManagers.get() && !rebuildingFolders;
}

public synchronized void resetFolders() {
Expand Down Expand Up @@ -319,7 +326,7 @@
private FolderManager getFolderManager(int tierLevel, String folderName)
throws DiskSpaceInsufficientException {
TierDirectory key = new TierDirectory(tierLevel, folderName);
TierFolderManager entry = folderManagers.get(key);
TierFolderManager entry = folderManagers.get().get(key);
if (entry == null) {
throw new IndexOutOfBoundsException(key.toString());
}
Expand Down Expand Up @@ -414,7 +421,7 @@
}

public int getTiersNum() {
return tierDiskTotalSpace.length;
return tierDiskTotalSpace.get().length;
}

public int getFileTierLevel(File file) {
Expand Down Expand Up @@ -445,7 +452,7 @@
}

public long[] getTierDiskTotalSpace() {
return Arrays.copyOf(tierDiskTotalSpace, tierDiskTotalSpace.length);
return tierDiskTotalSpace.get().clone();
}

public long[] getTierDiskUsableSpace() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,24 @@ public void tearDown() {
tierManager.resetFolders();
}

@Test
public void testTierDiskTotalSpaceFollowsTheTiersAndIsCopied() {
config.setTierDataDirs(
new String[][] {
{new File(temporaryFolder.getRoot(), "tier0").getAbsolutePath()},
{new File(temporaryFolder.getRoot(), "tier1").getAbsolutePath()}
});
tierManager.resetFolders();

long[] totalSpace = tierManager.getTierDiskTotalSpace();
assertEquals(2, totalSpace.length);
assertEquals(tierManager.getTiersNum(), totalSpace.length);
assertTrue(totalSpace[0] > 0 && totalSpace[1] > 0);

totalSpace[0] = -1;
assertTrue(tierManager.getTierDiskTotalSpace()[0] > 0);
}

@Test
public void testConcurrentResetFoldersAndGetNextFolder() throws Exception {
List<Callable<?>> folderAccessors =
Expand Down
Loading