Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ private <T extends InputFile> void scanBatch(BatchModeContext context, List<T> b
Set<Runnable> environmentsCleaners = new HashSet<>();
JParserConfig.Mode.BATCH
.create(javaVersion, context.getClasspath(), sonarComponents.shouldIgnoreUnnamedModuleForSplitPackage())
.withSourcepath(sonarComponents.getSourcepath())
.parse(batchFiles, sonarComponents::analysisCancelled, analysisProgress, (input, result) -> scanAsBatchCallback(input, result, context, environmentsCleaners));
// Due to a bug in ECJ, JAR files remain locked after the analysis on Windows, we unlock them manually, at the end of each batches. See SONARJAVA-3609.
environmentsCleaners.forEach(Runnable::run);
Expand Down
17 changes: 17 additions & 0 deletions java-frontend/src/main/java/org/sonar/java/SonarComponents.java
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,12 @@ public class SonarComponents extends CheckRegistrar.RegistrarContext {
*/
public static final String SONAR_FAIL_ON_STACKOVERFLOW = "sonar.java.internal.failOnStackOverflow";

/**
* Comma-separated directories whose Java files are read to resolve types, without being analyzed.
* Relative paths are resolved against the module base directory. Only the SemSitter bridge reads them.
*/
public static final String SONAR_SOURCEPATH_KEY = "sonar.java.internal.sourcepath";

private static final Version SONARLINT_6_3 = Version.parse("6.3");
private static final Version SONARQUBE_9_2 = Version.parse("9.2");
@VisibleForTesting
Expand Down Expand Up @@ -506,6 +512,17 @@ public boolean shouldFailOnStackOverflow() {
return context.config().getBoolean(SONAR_FAIL_ON_STACKOVERFLOW).orElse(true);
}

public List<File> getSourcepath() {
if (context == null || context.config() == null) {
return List.of();
}
return Stream.of(context.config().getStringArray(SONAR_SOURCEPATH_KEY))
.map(String::trim)
.filter(path -> !path.isEmpty())
.map(context.fileSystem()::resolvePath)
.toList();
}

private static long computeIdealBatchSize() {
// We take a fraction of the total memory available though -Xmx.
// If we assume that the average size of a file is 5KB and the average CI should have 1GB of memory,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ private void scan(Iterable<? extends InputFile> inputFiles, Consumer<Compilation
sonarComponents.shouldIgnoreUnnamedModuleForSplitPackage();
JParserConfig.Mode.FILE_BY_FILE
.create(visitor.getJavaVersion(), visitor.getClasspath(), shouldIgnoreUnnamedModuleForSplitPacakge)
.withSourcepath(sonarComponents != null ? sonarComponents.getSourcepath() : List.of())
.parse(filesNames,
this::analysisCancelled,
analysisProgress,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.sonar.java.semsitter.SemsitterBridge;
import java.util.function.BiConsumer;
import java.util.function.BooleanSupplier;
Expand All @@ -47,6 +51,7 @@ public abstract class JParserConfig {
final List<File> classpath;
final boolean shouldIgnoreUnnamedModuleForSplitPackage;
private SemsitterBridge bridge;
private List<File> sourcepath = List.of();

private JParserConfig(JavaVersion javaVersion, List<File> classpath, boolean shouldIgnoreUnnamedModuleForSplitPackage) {
this.javaVersion = javaVersion;
Expand All @@ -57,6 +62,14 @@ private JParserConfig(JavaVersion javaVersion, List<File> classpath, boolean sho
public abstract void parse(Iterable<? extends InputFile> inputFiles, BooleanSupplier isCanceled,
AnalysisProgress analysisProgress, BiConsumer<InputFile, Result> action);

/**
* Directories whose Java files are read to resolve types, without being analyzed.
*/
public JParserConfig withSourcepath(List<File> sourcepath) {
this.sourcepath = List.copyOf(sourcepath);
return this;
}

public enum Mode {
BATCH(Batch::new),
FILE_BY_FILE(FileByFile::new);
Expand Down Expand Up @@ -211,7 +224,7 @@ void openBridge(List<? extends InputFile> files) {
Path root = files.get(0).path().toAbsolutePath().getParent();
bridge.analyze(
root == null ? Path.of(".") : root,
files.stream().map(file -> file.path().toAbsolutePath()).toList(),
filesToScan(files.stream().map(file -> file.path().toAbsolutePath()).toList(), sourcepath),
jdkClasspath());
}
} catch (Exception failure) {
Expand All @@ -220,6 +233,47 @@ void openBridge(List<? extends InputFile> files) {
}
}

/**
* The files the bridge scans: the analyzed files, then the Java files under the sourcepath directories that are not
* analyzed. The bridge resolves types across all of them, but only the analyzed files are parsed and reach the rules.
*/
@VisibleForTesting
static List<Path> filesToScan(List<Path> analyzedFiles, List<File> sourcepath) {
if (sourcepath.isEmpty()) {
return analyzedFiles;
}
Set<Path> seen = new HashSet<>();
analyzedFiles.forEach(file -> seen.add(realPath(file)));
List<Path> result = new ArrayList<>(analyzedFiles);
for (File directory : sourcepath) {
if (!directory.isDirectory()) {
LOG.debug("Skipping sourcepath entry that is not a directory: {}", directory);
continue;
}
try (Stream<Path> walk = Files.walk(directory.toPath())) {
walk.filter(file -> file.toString().endsWith(".java") && Files.isRegularFile(file))
.map(Path::toAbsolutePath)
.forEach(file -> {
if (seen.add(realPath(file))) {
result.add(file);
}
});
} catch (IOException | UncheckedIOException e) {
LOG.warn("Could not read sourcepath directory {} ({})", directory, e.toString());
}
}
LOG.debug("SemSitter reads {} sourcepath files next to {} analyzed files", result.size() - analyzedFiles.size(), analyzedFiles.size());
return result;
}

private static Path realPath(Path file) {
try {
return file.toRealPath();
} catch (IOException e) {
return file.normalize();
}
}

static List<String> jdkClasspath() {
return List.of(Path.of(System.getProperty("java.home"), "lib", "modules").toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.CsvSource;
Expand Down Expand Up @@ -1140,6 +1141,24 @@ void shouldIgnoreUnnamedModuleForSplitPackage_returns_true_when_analysis_paramet
assertThat(sonarComponents.shouldIgnoreUnnamedModuleForSplitPackage()).isTrue();
}

@Test
void getSourcepath_is_empty_by_default() {
SonarComponents sonarComponents = new SonarComponents(null, null, null, null, null, null);
sonarComponents.setSensorContext(SensorContextTester.create(new File("")).setSettings(new MapSettings()));
assertThat(sonarComponents.getSourcepath()).isEmpty();
}

@Test
void getSourcepath_resolves_entries_against_the_base_directory(@TempDir File baseDir) throws IOException {
File absolute = new File(baseDir, "other/src/main/java").getAbsoluteFile();
MapSettings settings = new MapSettings().setProperty("sonar.java.internal.sourcepath", " src/main/java , ," + absolute.getPath());
SonarComponents sonarComponents = new SonarComponents(null, null, null, null, null, null);
sonarComponents.setSensorContext(SensorContextTester.create(baseDir).setSettings(settings));
assertThat(sonarComponents.getSourcepath())
.extracting(File::getCanonicalFile)
.containsExactly(new File(baseDir, "src/main/java").getCanonicalFile(), absolute.getCanonicalFile());
}

@Test
void shouldFailOnStackOverflow_returns_true_by_default() {
MapSettings settings = new MapSettings();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,24 @@
*/
package org.sonar.java.model;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.event.Level;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.java.AnalysisProgress;
import org.sonar.java.TestUtils;
import org.sonar.java.testing.ThreadLocalLogTester;
import org.sonar.plugins.java.api.semantic.Symbol;
import org.sonar.plugins.java.api.tree.BaseTreeVisitor;
import org.sonar.plugins.java.api.tree.MethodInvocationTree;

import static org.assertj.core.api.Assertions.assertThat;
import static org.sonar.java.model.JParserConfig.shouldEnablePreviewFlag;
Expand Down Expand Up @@ -55,4 +68,67 @@ void a_debug_message_is_logged_when_shouldIgnoreUnnamedModuleForSplitPackage_is_
JParserConfig.Mode.BATCH.create(new JavaVersionImpl(17), Collections.emptyList(), true);
assertThat(logTester.logs()).containsExactly("The Java analyzer will ignore the unnamed module for split packages.");
}

@Test
void files_to_scan_are_the_analyzed_files_when_there_is_no_sourcepath(@TempDir Path root) throws IOException {
Path analyzed = Files.writeString(root.resolve("A.java"), "class A {}");
assertThat(JParserConfig.filesToScan(List.of(analyzed), List.of())).containsExactly(analyzed);
}

@Test
void files_to_scan_add_the_java_files_of_the_sourcepath_once(@TempDir Path root) throws IOException {
Path sources = Files.createDirectories(root.resolve("src/main/java/p"));
Path analyzed = Files.writeString(sources.resolve("A.java"), "package p; class A {}");
Path other = Files.writeString(sources.resolve("B.java"), "package p; class B {}");
Path nested = Files.writeString(Files.createDirectories(sources.resolve("q")).resolve("C.java"), "package p.q; class C {}");
Files.writeString(sources.resolve("notes.txt"), "not java");
File sourceRoot = root.resolve("src/main/java").toFile();

List<Path> files = JParserConfig.filesToScan(List.of(analyzed), List.of(sourceRoot, sourceRoot, root.resolve("missing").toFile()));

assertThat(files.get(0)).isEqualTo(analyzed);
assertThat(files).containsExactlyInAnyOrder(analyzed, other.toAbsolutePath(), nested.toAbsolutePath());
}

@Test
void sourcepath_files_resolve_types_but_are_not_analyzed(@TempDir Path root) throws IOException {
Path sources = root.resolve("src/main/java");
Files.writeString(Files.createDirectories(sources.resolve("com/a")).resolve("Util.java"), """
package com.a;
public class Util {
public static String make() { return ""; }
}
""");
Path example = Files.writeString(Files.createDirectories(sources.resolve("com/b")).resolve("Example.java"), """
package com.b;
import com.a.Util;
class Example {
int m() { return Util.make().length(); }
}
""");
InputFile inputFile = TestUtils.inputFile(example.toFile());
List<InputFile> parsed = new ArrayList<>();
List<Symbol.MethodSymbol> invoked = new ArrayList<>();

JParserConfig.Mode.BATCH.create(new JavaVersionImpl(17), Collections.emptyList())
.withSourcepath(List.of(sources.toFile()))
.parse(List.of(inputFile), () -> false, new AnalysisProgress(1), (file, result) -> {
parsed.add(file);
try {
result.get().accept(new BaseTreeVisitor() {
@Override
public void visitMethodInvocation(MethodInvocationTree tree) {
invoked.add(tree.methodSymbol());
super.visitMethodInvocation(tree);
}
});
} catch (Exception e) {
throw new IllegalStateException(e);
}
});

assertThat(parsed).containsExactly(inputFile);
assertThat(invoked).extracting(symbol -> symbol.owner().type().fullyQualifiedName() + "#" + symbol.name())
.containsExactlyInAnyOrder("java.lang.String#length", "com.a.Util#make");
}
}
Loading