Skip to content

Commit 7fd8727

Browse files
committed
Extract the bundled formatter on first use
A class loader cannot load a jar inside a jar, and the IDE plugin needs files it can still find after a clean. BundledFormatter.jars extracts the plugin's formatter jars into <Gradle user home>/open-java-format-caches/<hash>/, where the hash is of jars.txt, which holds every jar's sha256: another formatter gets another directory, and a directory, once there, never changes. A daemon that holds the jars open, as Windows does not let a file be replaced, is in nobody's way. Builds and daemons may extract at the same time. Each copies the jars into a directory of its own next to the target, checking every jar against its sha256, and renames it into place, which fails when the target is already there; the build that loses deletes its copy and uses the one that won. There is no file lock: FileChannel.lock fails in a daemon whose other build holds the same file through another class loader of the plugin. A directory that was deleted, or lost a jar, is extracted again. A jar that does not match the list fails with its name and leaves no directory behind. Nothing calls this yet.
1 parent ec66207 commit 7fd8727

2 files changed

Lines changed: 329 additions & 1 deletion

File tree

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
15+
package com.palantir.javaformat.gradle;
16+
17+
import java.io.File;
18+
import java.io.IOException;
19+
import java.io.InputStream;
20+
import java.io.UncheckedIOException;
21+
import java.nio.charset.StandardCharsets;
22+
import java.nio.file.AtomicMoveNotSupportedException;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.nio.file.StandardCopyOption;
26+
import java.security.DigestInputStream;
27+
import java.security.MessageDigest;
28+
import java.security.NoSuchAlgorithmException;
29+
import java.util.Comparator;
30+
import java.util.HexFormat;
31+
import java.util.List;
32+
import java.util.stream.Stream;
33+
34+
/**
35+
* The Java-based formatter that comes inside the plugin: the jars of its runtime classpath, which the build puts under
36+
* {@code open-java-format/formatter/} with a list of them, {@code jars.txt}.
37+
*
38+
* <p>A class loader cannot load a jar inside a jar, and the IDE plugin needs files it can find after a clean, so the
39+
* jars are extracted on first use into {@code <Gradle user home>/open-java-format-caches/<hash>/}. The hash is of the
40+
* list, which holds every jar's sha256: another formatter is another directory, and a directory, once there, never
41+
* changes. That is also why a daemon holding the jars open, as Windows does not let a file be replaced, gets in nobody's
42+
* way.
43+
*
44+
* <p>Builds and daemons may extract at the same time. Each extracts into a directory of its own next to the target and
45+
* renames it into place, which fails when the target is already there; the build that loses deletes its copy and uses
46+
* the one that won.
47+
*/
48+
public final class BundledFormatter {
49+
50+
private static final String BUNDLE = "open-java-format/formatter/";
51+
private static final String LIST = BUNDLE + "jars.txt";
52+
static final String CACHES = "open-java-format-caches";
53+
54+
private BundledFormatter() {}
55+
56+
/** Names the directory the jars go in, and changes with every jar: the first 16 hex digits of the list's sha256. */
57+
public static String hash() {
58+
return hash(BundledFormatter.class.getClassLoader());
59+
}
60+
61+
/** The formatter's jars, in the order of its classpath, extracted under this Gradle user home if not there yet. */
62+
public static List<File> jars(File gradleUserHome) {
63+
return jars(gradleUserHome.toPath(), BundledFormatter.class.getClassLoader());
64+
}
65+
66+
static String hash(ClassLoader plugin) {
67+
return sha256(resource(plugin, LIST)).substring(0, 16);
68+
}
69+
70+
static List<File> jars(Path gradleUserHome, ClassLoader plugin) {
71+
var list = new String(resource(plugin, LIST), StandardCharsets.UTF_8)
72+
.lines()
73+
.map(Jar::parse)
74+
.toList();
75+
var caches = gradleUserHome.resolve(CACHES);
76+
var target = caches.resolve(hash(plugin));
77+
if (!isComplete(target, list)) {
78+
extract(plugin, list, caches, target);
79+
}
80+
return list.stream().map(jar -> target.resolve(jar.name()).toFile()).toList();
81+
}
82+
83+
private static boolean isComplete(Path dir, List<Jar> list) {
84+
return list.stream().allMatch(jar -> Files.isRegularFile(dir.resolve(jar.name())));
85+
}
86+
87+
private static void extract(ClassLoader plugin, List<Jar> list, Path caches, Path target) {
88+
try {
89+
Files.createDirectories(caches);
90+
var copy = Files.createTempDirectory(caches, target.getFileName() + "-");
91+
try {
92+
for (var jar : list) {
93+
copy(plugin, jar, copy.resolve(jar.name()));
94+
}
95+
if (Files.exists(target) && !isComplete(target, list)) {
96+
// Someone deleted a jar out of it. Whoever still runs from it keeps the files it has open.
97+
delete(target);
98+
}
99+
moveIntoPlace(copy, target);
100+
} finally {
101+
delete(copy);
102+
}
103+
} catch (IOException e) {
104+
throw new UncheckedIOException("Could not extract the formatter's jars into " + target, e);
105+
}
106+
}
107+
108+
private static void copy(ClassLoader plugin, Jar jar, Path file) throws IOException {
109+
var digest = sha256Digest();
110+
try (InputStream in = new DigestInputStream(open(plugin, BUNDLE + jar.name()), digest)) {
111+
Files.copy(in, file);
112+
}
113+
var actual = HexFormat.of().formatHex(digest.digest());
114+
if (!actual.equals(jar.sha256())) {
115+
throw new IllegalStateException(String.format(
116+
"%s in the open-java-format Gradle plugin has sha256 %s, but %s lists %s: the plugin's jar is"
117+
+ " damaged",
118+
jar.name(), actual, LIST, jar.sha256()));
119+
}
120+
}
121+
122+
/** Renames the copy into place, unless another build got there first: then the copy is left to be deleted. */
123+
private static void moveIntoPlace(Path copy, Path target) throws IOException {
124+
try {
125+
Files.move(copy, target, StandardCopyOption.ATOMIC_MOVE);
126+
} catch (AtomicMoveNotSupportedException e) {
127+
// Both are in the same directory, so the move is a rename all the same.
128+
moveUnlessThere(copy, target);
129+
} catch (IOException e) {
130+
if (!Files.isDirectory(target)) {
131+
throw e;
132+
}
133+
}
134+
}
135+
136+
private static void moveUnlessThere(Path copy, Path target) throws IOException {
137+
try {
138+
Files.move(copy, target);
139+
} catch (IOException e) {
140+
if (!Files.isDirectory(target)) {
141+
throw e;
142+
}
143+
}
144+
}
145+
146+
private static void delete(Path dir) throws IOException {
147+
if (!Files.exists(dir)) {
148+
return;
149+
}
150+
try (Stream<Path> paths = Files.walk(dir)) {
151+
for (var path : paths.sorted(Comparator.reverseOrder()).toList()) {
152+
Files.deleteIfExists(path);
153+
}
154+
}
155+
}
156+
157+
private static byte[] resource(ClassLoader plugin, String name) {
158+
try (InputStream in = open(plugin, name)) {
159+
return in.readAllBytes();
160+
} catch (IOException e) {
161+
throw new UncheckedIOException("Could not read " + name + " from the open-java-format Gradle plugin", e);
162+
}
163+
}
164+
165+
private static InputStream open(ClassLoader plugin, String name) throws IOException {
166+
var in = plugin.getResourceAsStream(name);
167+
if (in == null) {
168+
throw new IOException(name + " is not in the open-java-format Gradle plugin");
169+
}
170+
return in;
171+
}
172+
173+
private static String sha256(byte[] bytes) {
174+
return HexFormat.of().formatHex(sha256Digest().digest(bytes));
175+
}
176+
177+
private static MessageDigest sha256Digest() {
178+
try {
179+
return MessageDigest.getInstance("SHA-256");
180+
} catch (NoSuchAlgorithmException e) {
181+
throw new IllegalStateException("Every JVM has SHA-256", e);
182+
}
183+
}
184+
185+
/** A line of jars.txt: the jar's sha256 and its name. */
186+
private record Jar(String sha256, String name) {
187+
static Jar parse(String line) {
188+
var space = line.indexOf(' ');
189+
return new Jar(line.substring(0, space), line.substring(space + 1));
190+
}
191+
}
192+
}

‎gradle-open-java-format/src/test/java/com/palantir/javaformat/gradle/BundledFormatterTest.java‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,29 +15,46 @@
1515
package com.palantir.javaformat.gradle;
1616

1717
import static org.assertj.core.api.Assertions.assertThat;
18+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
1819

1920
import java.io.File;
2021
import java.io.IOException;
2122
import java.io.InputStream;
2223
import java.io.UncheckedIOException;
24+
import java.net.URL;
25+
import java.net.URLClassLoader;
2326
import java.nio.charset.StandardCharsets;
2427
import java.nio.file.Files;
2528
import java.nio.file.Path;
29+
import java.nio.file.attribute.FileTime;
2630
import java.security.MessageDigest;
2731
import java.security.NoSuchAlgorithmException;
32+
import java.util.ArrayList;
2833
import java.util.Arrays;
34+
import java.util.Comparator;
2935
import java.util.HexFormat;
3036
import java.util.List;
3137
import java.util.Objects;
38+
import java.util.concurrent.CompletableFuture;
39+
import java.util.concurrent.CountDownLatch;
40+
import java.util.concurrent.Executors;
41+
import java.util.stream.Stream;
42+
import org.junit.jupiter.api.RepeatedTest;
3243
import org.junit.jupiter.api.Test;
44+
import org.junit.jupiter.api.io.TempDir;
3345

3446
final class BundledFormatterTest {
3547

36-
/** ./gradlew writeImplClasspath generates this file: the formatter's runtime classpath, as bundleFormatter takes it. */
48+
/**
49+
* ./gradlew writeImplClasspath generates this file: the formatter's runtime classpath, as bundleFormatter takes it.
50+
*/
3751
private static final Path CLASSPATH_FILE = Path.of("build/impl.classpath");
3852

3953
private static final String BUNDLE = "open-java-format/formatter/";
4054

55+
@TempDir
56+
private Path gradleUserHome;
57+
4158
@Test
4259
void the_plugin_carries_the_jars_of_the_formatter_runtime_classpath() throws IOException {
4360
var classpath = Arrays.stream(Files.readString(CLASSPATH_FILE).trim().split(File.pathSeparator))
@@ -61,12 +78,131 @@ void every_jar_in_the_list_is_in_the_plugin_as_listed() {
6178
}
6279
}
6380

81+
@Test
82+
void the_jars_are_extracted_into_a_directory_named_by_the_hash_once() throws IOException {
83+
var jars = BundledFormatter.jars(gradleUserHome.toFile());
84+
85+
var dir = gradleUserHome.resolve("open-java-format-caches").resolve(BundledFormatter.hash());
86+
assertThat(BundledFormatter.hash()).matches("[0-9a-f]{16}");
87+
assertThat(jars).extracting(File::getName).containsExactlyElementsOf(bundledNames());
88+
assertThat(jars).allSatisfy(jar -> assertThat(jar.toPath().getParent()).isEqualTo(dir));
89+
for (var line : bundledList()) {
90+
var name = line.substring(line.indexOf(' ') + 1);
91+
assertThat(sha256(read(dir.resolve(name)))).as(name).isEqualTo(line.substring(0, line.indexOf(' ')));
92+
}
93+
94+
// A second build finds them there and leaves them as they are: a daemon may have them open.
95+
var written = lastModified(jars);
96+
assertThat(BundledFormatter.jars(gradleUserHome.toFile())).isEqualTo(jars);
97+
assertThat(lastModified(jars)).isEqualTo(written);
98+
}
99+
100+
// Repeated, so that the builds do race to rename their copies into place.
101+
@RepeatedTest(5)
102+
void builds_that_extract_at_the_same_time_get_the_same_jars() throws Exception {
103+
var executor = Executors.newFixedThreadPool(4);
104+
try {
105+
var start = new CountDownLatch(1);
106+
var builds = Stream.generate(() -> CompletableFuture.supplyAsync(
107+
() -> {
108+
await(start);
109+
return BundledFormatter.jars(gradleUserHome.toFile());
110+
},
111+
executor))
112+
.limit(4)
113+
.toList();
114+
start.countDown();
115+
116+
var results = builds.stream().map(CompletableFuture::join).toList();
117+
118+
assertThat(results).allSatisfy(jars -> assertThat(jars).isEqualTo(results.get(0)));
119+
assertThat(results.get(0)).allSatisfy(jar -> assertThat(jar).isFile());
120+
// The copies of the builds that lost are gone.
121+
assertThat(list(gradleUserHome.resolve("open-java-format-caches")))
122+
.containsExactly(BundledFormatter.hash());
123+
} finally {
124+
executor.shutdownNow();
125+
}
126+
}
127+
128+
@Test
129+
void a_deleted_directory_is_extracted_again() throws IOException {
130+
var jars = BundledFormatter.jars(gradleUserHome.toFile());
131+
delete(gradleUserHome.resolve("open-java-format-caches"));
132+
133+
assertThat(BundledFormatter.jars(gradleUserHome.toFile())).isEqualTo(jars);
134+
assertThat(jars).allSatisfy(jar -> assertThat(jar).isFile());
135+
}
136+
137+
@Test
138+
void a_directory_that_lost_a_jar_is_extracted_again() throws IOException {
139+
var jars = BundledFormatter.jars(gradleUserHome.toFile());
140+
Files.delete(jars.get(0).toPath());
141+
142+
assertThat(BundledFormatter.jars(gradleUserHome.toFile())).isEqualTo(jars);
143+
assertThat(jars).allSatisfy(jar -> assertThat(jar).isFile());
144+
}
145+
146+
@Test
147+
void a_jar_that_does_not_match_the_list_fails_and_leaves_nothing_behind(@TempDir Path plugin) throws IOException {
148+
var bundle = Files.createDirectories(plugin.resolve(BUNDLE));
149+
Files.writeString(bundle.resolve("formatter.jar"), "not what the list says");
150+
Files.writeString(
151+
bundle.resolve("jars.txt"),
152+
sha256("what the list says".getBytes(StandardCharsets.UTF_8)) + " formatter.jar\n");
153+
154+
try (var loader = new URLClassLoader(new URL[] {plugin.toUri().toURL()}, null)) {
155+
assertThatThrownBy(() -> BundledFormatter.jars(gradleUserHome, loader))
156+
.isInstanceOf(IllegalStateException.class)
157+
.hasMessageContaining("formatter.jar")
158+
.hasMessageContaining("damaged");
159+
}
160+
assertThat(list(gradleUserHome.resolve("open-java-format-caches"))).isEmpty();
161+
}
162+
64163
private static List<String> bundledList() {
65164
return new String(resource(BUNDLE + "jars.txt"), StandardCharsets.UTF_8)
66165
.lines()
67166
.toList();
68167
}
69168

169+
private static List<String> bundledNames() {
170+
return bundledList().stream()
171+
.map(line -> line.substring(line.indexOf(' ') + 1))
172+
.toList();
173+
}
174+
175+
private static List<FileTime> lastModified(List<File> jars) throws IOException {
176+
var times = new ArrayList<FileTime>();
177+
for (var jar : jars) {
178+
times.add(Files.getLastModifiedTime(jar.toPath()));
179+
}
180+
return times;
181+
}
182+
183+
private static List<String> list(Path dir) throws IOException {
184+
try (Stream<Path> paths = Files.list(dir)) {
185+
return paths.map(path -> path.getFileName().toString()).toList();
186+
}
187+
}
188+
189+
private static void delete(Path dir) throws IOException {
190+
try (Stream<Path> paths = Files.walk(dir)) {
191+
for (var path : paths.sorted(Comparator.reverseOrder()).toList()) {
192+
Files.delete(path);
193+
}
194+
}
195+
}
196+
197+
private static void await(CountDownLatch latch) {
198+
try {
199+
latch.await();
200+
} catch (InterruptedException e) {
201+
Thread.currentThread().interrupt();
202+
throw new IllegalStateException(e);
203+
}
204+
}
205+
70206
private static byte[] resource(String name) {
71207
try (InputStream in = BundledFormatterTest.class.getClassLoader().getResourceAsStream(name)) {
72208
return Objects.requireNonNull(in, name + " is not on the classpath").readAllBytes();

0 commit comments

Comments
 (0)