diff --git a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/access/AsyncApiAccess.java b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/access/AsyncApiAccess.java index 362f7b83f4..e9b4355bd1 100644 --- a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/access/AsyncApiAccess.java +++ b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/access/AsyncApiAccess.java @@ -79,11 +79,12 @@ public static AsyncApiDocument parseFromText(String schemaText) { on AsyncApiDocument.getWarnings(), and where those warnings go is the caller's decision. */ private static AsyncApiDocument parse(String schemaText, DocumentLocation location) { - return AsyncApiParser.parse(schemaText, location); + return AsyncApiParser.parse(schemaText, location, AsyncApiAccess::fetch); } /** - * Read the text of a document, wherever it lives. + * Read the text of a document, wherever it lives. Also used to follow references to other + * documents, which is why it takes the kind of location explicitly. */ private static String fetch(String location, DocumentLocationType type) { diff --git a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/models/DocumentLocation.java b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/models/DocumentLocation.java index 1bb5267ba8..cc03c67a6f 100644 --- a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/models/DocumentLocation.java +++ b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/models/DocumentLocation.java @@ -1,5 +1,6 @@ package com.webfuzzing.asyncapi.models; +import java.util.Locale; import java.util.Objects; /** @@ -17,6 +18,11 @@ public class DocumentLocation { */ public static final DocumentLocation MEMORY = new DocumentLocation("", DocumentLocationType.MEMORY); + /** + * How a file on disk looks when written as a URL rather than as a path. + */ + private static final String FILE_URL_PREFIX = "file:"; + private final String location; private final DocumentLocationType type; @@ -46,6 +52,17 @@ public DocumentLocationType getType() { return type; } + /** + * Whether this is a path on the file system written as a path, rather than as a + * {@code file:} URL. Both are read from disk, but only the former is not a URI, so a + * relative reference is resolved against it through the file system rather than per + * RFC 3986. + */ + public boolean isPlainFilePath() { + return type == DocumentLocationType.LOCAL + && !location.toLowerCase(Locale.ENGLISH).startsWith(FILE_URL_PREFIX); + } + @Override public boolean equals(Object other) { if (this == other) { diff --git a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/parser/AsyncApiParser.java b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/parser/AsyncApiParser.java index fee2bea33d..f17810c5ac 100644 --- a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/parser/AsyncApiParser.java +++ b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/parser/AsyncApiParser.java @@ -11,6 +11,7 @@ import com.webfuzzing.asyncapi.models.AsyncApiOperation; import com.webfuzzing.asyncapi.models.AsyncApiReply; import com.webfuzzing.asyncapi.models.DocumentLocation; +import com.webfuzzing.asyncapi.resolver.AsyncApiDocumentFetcher; import com.webfuzzing.asyncapi.resolver.AsyncApiRefResolver; import java.util.ArrayDeque; @@ -75,9 +76,12 @@ private AsyncApiParser() { } /** - * Parse {@code schemaText}, which was retrieved from {@code location}. + * Parse {@code schemaText}, reaching for any document it refers to with {@code fetch}. */ - public static AsyncApiDocument parse(String schemaText, DocumentLocation location) { + public static AsyncApiDocument parse( + String schemaText, + DocumentLocation location, + AsyncApiDocumentFetcher fetch) { JsonNode root; try { @@ -107,6 +111,9 @@ public static AsyncApiDocument parse(String schemaText, DocumentLocation locatio List warnings = new ArrayList<>(); + AsyncApiRefResolver.inlineExternalDocuments( + (ObjectNode) root, location, warnings, fetch, AsyncApiMapper::readTree); + String defaultContentType = scalarOf(root.get("defaultContentType")); if (defaultContentType == null) { defaultContentType = AsyncApiDocument.DEFAULT_CONTENT_TYPE; diff --git a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiDocumentFetcher.java b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiDocumentFetcher.java new file mode 100644 index 0000000000..802ecad87a --- /dev/null +++ b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiDocumentFetcher.java @@ -0,0 +1,24 @@ +package com.webfuzzing.asyncapi.resolver; + +import com.webfuzzing.asyncapi.models.DocumentLocationType; + +/** + * How the parser gets hold of a document it does not already have: given an absolute location + * and how to read it, hand back the text. + * + * Kept as an interface so that the resolver does not depend on the retrieval code, and so tests + * can supply documents without any I/O. + */ +@FunctionalInterface +public interface AsyncApiDocumentFetcher { + + /** + * @param location an absolute location, already resolved against the referring document + * @param type how that location is to be read + * @return the text of the document + * @throws RuntimeException if the document cannot be retrieved. The caller turns that into + * a warning rather than letting it escape, as one unreachable + * document costs only what refers to it. + */ + String fetch(String location, DocumentLocationType type); +} diff --git a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiRefResolver.java b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiRefResolver.java index a703d8a102..7f59f22cdc 100644 --- a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiRefResolver.java +++ b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/AsyncApiRefResolver.java @@ -1,11 +1,25 @@ package com.webfuzzing.asyncapi.resolver; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.webfuzzing.asyncapi.models.DocumentLocation; +import com.webfuzzing.asyncapi.models.DocumentLocationType; +import java.io.IOException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; /** * {@code $ref} handling for AsyncAPI documents. @@ -19,13 +33,61 @@ * References inside a JSON Schema are not resolved at all, because whatever consumes * them later would rather resolve them itself. What is done instead is to make sure it can: * every schema a payload can reach has to sit in one flat map under - * {@code #/components/schemas/}, which is what {@link #schemaKeyOf} is for. + * {@code #/components/schemas/}. That is why a document referenced from another is + * inlined by {@link #inlineExternalDocuments} rather than merely linked -- its schemas + * are copied in under a namespaced key, and the pointers that referred to it are rewritten to + * match. */ public class AsyncApiRefResolver { private static final String REF = "$ref"; - public static final String SCHEMA_PREFIX = "#/components/schemas/"; + private static final String COMPONENTS = "components"; + + private static final String SCHEMAS = "schemas"; + + private static final String MESSAGES = "messages"; + + /** + * How a pointer at any component begins, e.g. the "#/components/" of + * "#/components/schemas/Order". + */ + private static final String COMPONENT_PREFIX = RefLocations.FRAGMENT_SEPARATOR + + RefLocations.PATH_SEPARATOR + COMPONENTS + RefLocations.PATH_SEPARATOR; + + public static final String SCHEMA_PREFIX = COMPONENT_PREFIX + SCHEMAS + RefLocations.PATH_SEPARATOR; + + /** + * The two component kinds that are worth pulling out of an external document. Schemas are + * the point of the exercise; messages come along because a document that splits its schemas + * out often splits its messages out too. + */ + private static final List INLINABLE = Arrays.asList(SCHEMAS, MESSAGES); + + /** + * A ceiling on how many other documents one document may drag in. Documents reference each + * other by path, and a server that answers every path -- or a symlink loop -- would + * otherwise be followed forever. + */ + private static final int MAX_IMPORTED_DOCUMENTS = 100; + + /** + * JSON Pointer escaping: a "/" inside a key is written "~1", and a "~" is written "~0". + */ + private static final String ESCAPED_SLASH = "~1"; + + private static final String TILDE = "~"; + + private static final String ESCAPED_TILDE = TILDE + "0"; + + /** + * How the text of a retrieved document is turned into a tree. Supplied by the caller so + * that this class does not have to know which reader is in use. + */ + @FunctionalInterface + public interface DocumentReader { + JsonNode readTree(String text) throws IOException; + } private AsyncApiRefResolver() { } @@ -46,7 +108,8 @@ public static JsonNode resolveLocal(JsonNode root, String ref) { JsonNode current = root; - for (String segment : ref.substring(1).split("/")) { + for (String segment : ref.substring(RefLocations.FRAGMENT_SEPARATOR.length()) + .split(RefLocations.PATH_SEPARATOR)) { if (segment.isEmpty()) { continue; @@ -77,7 +140,7 @@ public static String refKey(String ref, String expectedPrefix) { String key = ref.substring(expectedPrefix.length()); //must be a single segment: a deeper pointer is something else than what was asked for - if (key.trim().isEmpty() || key.contains("/")) { + if (key.trim().isEmpty() || key.contains(RefLocations.PATH_SEPARATOR)) { return null; } @@ -112,12 +175,143 @@ public static String schemaKeyOf(String ref) { } String rest = ref.substring(SCHEMA_PREFIX.length()); - int slash = rest.indexOf('/'); + int slash = rest.indexOf(RefLocations.PATH_SEPARATOR); String key = slash < 0 ? rest : rest.substring(0, slash); return key.trim().isEmpty() ? null : decodePointerSegment(key); } + /** + * Copy into {@code root} every schema (and message) that its {@code $ref} reach in other + * documents, so that afterwards the document is self-contained and every reference in it is + * local. + * + * Imported components are keyed {@code _ext__}, the hash being derived + * from the source location. That keeps them from colliding with the primary document's own + * components, and keeps the key stable across runs so generated output stays diffable. + * + * Documents referenced from documents that were themselves imported are followed too. + * Anything that cannot be retrieved, or that is referenced in a way not supported here, is + * reported in {@code warnings} and left alone: the affected message becomes unusable, but + * the rest of the document is still perfectly usable. + */ + public static void inlineExternalDocuments( + ObjectNode root, + DocumentLocation primaryLocation, + List warnings, + AsyncApiDocumentFetcher fetch, + DocumentReader reader) { + + if (primaryLocation.getType() == DocumentLocationType.MEMORY) { + //nothing to resolve against: a document given as text has no neighbours + List external = externalRefsOf(root); + if (!external.isEmpty()) { + warnings.add( + "The document was supplied as text, so its " + external.size() + " reference(s)" + + " to other documents cannot be resolved, e.g. '" + external.get(0) + "'"); + } + return; + } + + //key is the absolute location of an imported document, value is that document + Map loaded = new LinkedHashMap<>(); + + //locations already dealt with, whether imported or failed + Set settled = new LinkedHashSet<>(); + + //a document naming itself still needs its references turned back into local ones + boolean namesItself = false; + + //breadth-first, since an imported document may import further documents itself + Deque pending = new ArrayDeque<>(); + pending.add(new PendingDocument(root, primaryLocation)); + + while (!pending.isEmpty()) { + + PendingDocument current = pending.removeFirst(); + + for (String ref : externalRefsOf(current.root)) { + + String absolute = locationOf(ref, current.location, warnings); + + if (absolute == null) { + continue; + } + + if (absolute.equals(primaryLocation.getLocation())) { + //not another document at all: this one, named the long way + namesItself = true; + continue; + } + + if (!settled.add(absolute)) { + continue; + } + + if (loaded.size() >= MAX_IMPORTED_DOCUMENTS) { + warnings.add( + "More than " + MAX_IMPORTED_DOCUMENTS + " documents are referenced from this" + + " one. '" + ref + "' and any further reference are ignored."); + continue; + } + + DocumentLocationType type = locationTypeOf(absolute, current.location); + + JsonNode other; + try { + other = reader.readTree(fetch.fetch(absolute, type)); + } catch (Exception e) { + warnings.add( + "Failed to retrieve the document referenced as '" + ref + "': " + e.getMessage()); + continue; + } + + DocumentLocation otherLocation = new DocumentLocation(absolute, type); + loaded.put(absolute, new LoadedDocument(other, prefixFor(absolute), otherLocation)); + pending.add(new PendingDocument(other, otherLocation)); + } + } + + if (loaded.isEmpty() && !namesItself) { + return; + } + + /* + Order matters here. The primary document is rewritten first, while its tree still + holds only its own nodes: once the imported ones have been copied in, a walk of the + primary tree would reach them too and would resolve their relative references + against the wrong document -- quietly binding them to whatever happened to be there. + */ + rewriteRefs(root, primaryLocation, primaryLocation.getLocation(), null, loaded, warnings); + + for (LoadedDocument doc : loaded.values()) { + for (String kind : INLINABLE) { + JsonNode components = componentsOf(doc.root, kind); + if (components == null) { + continue; + } + for (JsonNode value : components) { + rewriteRefs(value, doc.location, primaryLocation.getLocation(), doc.prefix, loaded, warnings); + } + } + } + + for (LoadedDocument doc : loaded.values()) { + for (String kind : INLINABLE) { + JsonNode imported = componentsOf(doc.root, kind); + if (imported == null) { + continue; + } + ObjectNode target = ensureObject(ensureObject(root, COMPONENTS), kind); + Iterator> fields = imported.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + target.set(doc.prefix + field.getKey(), field.getValue()); + } + } + } + } + /** * Every {@code $ref} value under {@code node}, at any depth. */ @@ -127,6 +321,271 @@ public static List collectRefs(JsonNode node) { return refs; } + /** + * The distinct references under {@code node} that lead outside the document holding it. + */ + private static List externalRefsOf(JsonNode node) { + + Set external = new LinkedHashSet<>(); + + for (String ref : collectRefs(node)) { + if (!isLocal(ref)) { + external.add(ref); + } + } + + return new ArrayList<>(external); + } + + /** + * Where a reference points, as an absolute location. + * + * The resolution can throw on input it was not written for -- a protocol-relative + * {@code //host/path} reference read from a plain file path, for one. Nothing here is worth + * failing a whole document over. + */ + private static String locationOf(String ref, DocumentLocation from, List warnings) { + try { + return RefLocations.resolveDocumentLocation(ref, from, warnings); + } catch (Exception e) { + warnings.add("Cannot work out what document '" + ref + "' refers to: " + e.getMessage()); + return null; + } + } + + /** + * One document pulled in from elsewhere. + */ + private static class LoadedDocument { + + private final JsonNode root; + private final String prefix; + private final DocumentLocation location; + + private LoadedDocument(JsonNode root, String prefix, DocumentLocation location) { + this.root = root; + this.prefix = prefix; + this.location = location; + } + } + + /** + * A document whose references have not been walked yet. + */ + private static class PendingDocument { + + private final JsonNode root; + private final DocumentLocation location; + + private PendingDocument(JsonNode root, DocumentLocation location) { + this.root = root; + this.location = location; + } + } + + /** + * Point every {@code $ref} under {@code node} at the inlined copy of what it referred to. + * + * {@code base} is the document the node belongs to, which is what a relative reference is + * relative to. {@code ownerPrefix} is set only when the node came from an imported document: + * a plain {@code #/components/schemas/X} written in there means that document's X, + * which is now stored under the document's own prefix. The primary document's own local + * references are already correct and are left alone. + * + * {@code loaded} is keyed by the absolute location of each imported document, its value + * being that document and the prefix its components were copied in under. + */ + private static void rewriteRefs( + JsonNode node, + DocumentLocation base, + String primary, + String ownerPrefix, + Map loaded, + List warnings) { + + if (node.isObject()) { + + ObjectNode obj = (ObjectNode) node; + String ref = refOf(obj); + + if (ref != null) { + String rewritten = rewrite(ref, base, primary, ownerPrefix, loaded, warnings); + if (rewritten != null) { + obj.put(REF, rewritten); + } + } + + for (JsonNode value : obj) { + rewriteRefs(value, base, primary, ownerPrefix, loaded, warnings); + } + + } else if (node.isArray()) { + for (JsonNode value : node) { + rewriteRefs(value, base, primary, ownerPrefix, loaded, warnings); + } + } + } + + /** + * The new value for a single {@code $ref}, or null when it needs no change. + */ + private static String rewrite( + String ref, + DocumentLocation base, + String primary, + String ownerPrefix, + Map loaded, + List warnings) { + + if (isLocal(ref)) { + //a local reference inside an imported document now has to name the imported copy + return ownerPrefix == null ? null : renameComponent(fragmentOf(ref), ownerPrefix, ref, warnings); + } + + String absolute = locationOf(ref, base, warnings); + + if (absolute == null) { + return null; + } + + /* + Some generators write even a reference that stays inside the document as an absolute + one. It names this very file, so it is just a local reference written the long way, + and turning it back into one is all that is needed. + */ + if (absolute.equals(primary)) { + String fragment = fragmentOf(ref); + return fragment.trim().isEmpty() ? null : RefLocations.FRAGMENT_SEPARATOR + fragment; + } + + LoadedDocument target = loaded.get(absolute); + + if (target == null) { + return null; + } + + String fragment = fragmentOf(ref); + + if (fragment.trim().isEmpty()) { + warnings.add( + "Reference '" + ref + "' points at a whole document rather than at a component of" + + " it, which is not supported"); + return null; + } + + return renameComponent(fragment, target.prefix, ref, warnings); + } + + /** + * Whatever follows the first {@code #} of a reference, empty when there is none. + */ + private static String fragmentOf(String ref) { + int hash = ref.indexOf(RefLocations.FRAGMENT_SEPARATOR); + return hash < 0 ? "" : ref.substring(hash + 1); + } + + /** + * Turn the fragment of a reference into a local pointer at the inlined copy. + * + * Only the component's own key is renamed. A pointer may go deeper than the component -- + * {@code #/components/schemas/Order/properties/item} addresses one property of a schema -- + * and everything past the key describes a way *into* the imported node, which the rename + * must carry through untouched. Dropping the tail instead would leave the reference naming + * the primary document's own component of that name, silently binding a payload to an + * unrelated schema. + */ + private static String renameComponent( + String fragment, + String prefix, + String original, + List warnings) { + + String path = fragment.startsWith(RefLocations.PATH_SEPARATOR) + ? fragment.substring(RefLocations.PATH_SEPARATOR.length()) + : fragment; + String[] segments = path.split(RefLocations.PATH_SEPARATOR, -1); + + if (segments.length < 3 || !COMPONENTS.equals(segments[0]) || !INLINABLE.contains(segments[1])) { + warnings.add( + "Reference '" + original + "' points at '" + path + "' of another document." + + " Only " + COMPONENTS + RefLocations.PATH_SEPARATOR + SCHEMAS + + " and " + COMPONENTS + RefLocations.PATH_SEPARATOR + MESSAGES + + " can be imported."); + return null; + } + + StringBuilder renamed = new StringBuilder(COMPONENT_PREFIX) + .append(segments[1]).append(RefLocations.PATH_SEPARATOR) + .append(prefix).append(segments[2]); + + for (int i = 3; i < segments.length; i++) { + renamed.append(RefLocations.PATH_SEPARATOR).append(segments[i]); + } + + return renamed.toString(); + } + + /** + * A short, deterministic namespace for the components of one external document, so that + * they cannot collide with the primary document's own. + */ + private static String prefixFor(String absoluteLocation) { + + byte[] digest; + try { + digest = MessageDigest.getInstance("SHA-1") + .digest(absoluteLocation.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException e) { + //SHA-1 is required of every JVM, so this cannot happen + throw new IllegalStateException(e); + } + + StringBuilder hex = new StringBuilder(); + for (int i = 0; i < 4; i++) { + hex.append(String.format("%02x", digest[i])); + } + + return "_ext_" + hex + "_"; + } + + private static JsonNode componentsOf(JsonNode root, String kind) { + + JsonNode components = root.get(COMPONENTS); + + if (components == null) { + return null; + } + + JsonNode of = components.get(kind); + + return of != null && of.isObject() ? of : null; + } + + private static ObjectNode ensureObject(ObjectNode parent, String field) { + + JsonNode existing = parent.get(field); + + if (existing != null && existing.isObject()) { + return (ObjectNode) existing; + } + + return parent.putObject(field); + } + + private static DocumentLocationType locationTypeOf(String absoluteLocation, DocumentLocation from) { + + if (RefLocations.isHttpLocation(absoluteLocation)) { + return DocumentLocationType.REMOTE; + } + + //a document read off the classpath can only reference other classpath documents + if (from.getType() == DocumentLocationType.RESOURCE) { + return DocumentLocationType.RESOURCE; + } + + return DocumentLocationType.LOCAL; + } + private static void collectRefsInto(JsonNode node, List out) { if (node.isObject()) { @@ -165,6 +624,6 @@ private static String decodePointerSegment(String segment) { } } - return decoded.replace("~1", "/").replace("~0", "~"); + return decoded.replace(ESCAPED_SLASH, RefLocations.PATH_SEPARATOR).replace(ESCAPED_TILDE, TILDE); } } diff --git a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/RefLocations.java b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/RefLocations.java index 7d2928e010..9b859d4285 100644 --- a/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/RefLocations.java +++ b/core-extra/asyncapi-parser/src/main/java/com/webfuzzing/asyncapi/resolver/RefLocations.java @@ -1,24 +1,176 @@ package com.webfuzzing.asyncapi.resolver; +import com.webfuzzing.asyncapi.models.DocumentLocation; +import com.webfuzzing.asyncapi.models.DocumentLocationType; + +import java.net.URI; +import java.nio.file.Paths; +import java.util.List; +import java.util.Locale; + /** - * Working out what a {@code $ref} points at. + * Working out where the document a {@code $ref} points at actually lives. * - * A reference is a location followed by a {@code #} and a JSON Pointer. An empty location means - * the document making the reference, which is the only kind that can be followed without - * retrieving anything. + * A reference is a location followed by a {@code #} and a JSON Pointer, and the location part + * is written relative to the document making the reference. Turning that into something that + * can be retrieved is all this does. * * @see * Reference Object */ public class RefLocations { + /** + * Separates the document location from the JSON Pointer inside a {@code $ref}. + */ + public static final String FRAGMENT_SEPARATOR = "#"; + + /** + * Separates the segments of a JSON Pointer, and of a path. + */ + public static final String PATH_SEPARATOR = "/"; + + /** + * Separates the protocol from the rest of a location, as in "https:". + */ + private static final String PROTOCOL_SEPARATOR = ":"; + + /** + * A location naming a host but no protocol, which borrows the protocol of the document + * referring to it. + */ + private static final String PROTOCOL_RELATIVE_PREFIX = "//"; + + /** + * The schemes whose locations are already absolute, so need no resolving. + */ + private static final String HTTP_SCHEME = "http"; + + private static final String HTTPS_SCHEME = "https"; + + private static final String HTTP_PREFIX = HTTP_SCHEME + PROTOCOL_SEPARATOR; + + private static final String HTTPS_PREFIX = HTTPS_SCHEME + PROTOCOL_SEPARATOR; + private RefLocations() { } + /** + * Whether the location is an absolute http(s) URL, and so is already where the document + * lives rather than something to resolve against the document referring to it. + */ + public static boolean isHttpLocation(String location) { + + String lower = location.toLowerCase(Locale.ENGLISH); + + return lower.startsWith(HTTP_PREFIX) || lower.startsWith(HTTPS_PREFIX); + } + /** * Whether the reference stays inside the document making it. */ public static boolean isLocalRef(String ref) { - return ref.startsWith("#"); + return ref.startsWith(FRAGMENT_SEPARATOR); + } + + /** + * The absolute location of the document a reference points at, resolved against the + * document making the reference. Null when the reference is not one this can make sense of, + * in which case {@code messages} says why. + * + * @throws IllegalArgumentException if the referring document was supplied as text, as there + * is then nothing for a relative location to be relative to + */ + public static String resolveDocumentLocation(String ref, DocumentLocation currentSource, List messages) { + + String rawLocation = extractLocationPart(ref, messages); + + if (rawLocation == null) { + return null; + } + + if (isHttpLocation(rawLocation)) { + //location is absolute, so no need to do anything + return rawLocation; + } + + if (currentSource.getType() == DocumentLocationType.MEMORY) { + throw new IllegalArgumentException( + "Can't handle relative location for memory files: " + rawLocation); + } + + String csl = currentSource.getLocation(); + + if (rawLocation.startsWith(PROTOCOL_RELATIVE_PREFIX)) { + //as per specs, use same protocol as source + int separator = csl.indexOf(PROTOCOL_SEPARATOR); + if (separator < 0) { + /* + A protocol-relative reference read from something that has no protocol, such + as a plain file path. There is nothing to borrow, so the reference cannot be + resolved. + */ + messages.add("No protocol can be inferred for " + rawLocation + " from " + csl); + return null; + } + return csl.substring(0, separator) + PROTOCOL_SEPARATOR + rawLocation; + } + + //if we arrive here, it is a relative location + return resolveRelative(rawLocation, currentSource, messages); + } + + /** + * Resolve a relative location against the folder holding the document that refers to it. + * + * A URL, a {@code file:} URL and a classpath path are resolved as URIs, per RFC 3986. A + * plain file path is resolved through the file system instead: it may contain a space, or + * on Windows backslashes and a drive letter, none of which {@link URI} accepts. + * + * Both collapse "." and ".." segments, so two references to the same document produce the + * same string. That matters because the string is what tells imported documents apart. + */ + private static String resolveRelative( + String rawLocation, + DocumentLocation currentSource, + List messages) { + + String csl = currentSource.getLocation(); + + if (currentSource.isPlainFilePath()) { + return Paths.get(csl).resolveSibling(rawLocation).normalize().toString(); + } + + try { + return URI.create(csl).resolve(rawLocation).toString(); + } catch (IllegalArgumentException e) { + /* + The referring document's location is a URL or classpath path that is not a legal + URI -- one written with a space in it, say. There is then no well-defined way to + resolve a relative reference from it, so this says why rather than guessing. + */ + messages.add( + "Cannot resolve '" + rawLocation + "' against '" + csl + "', which is not a valid" + + " URI: " + e.getMessage()); + return null; + } + } + + /** + * The location part of a reference, i.e. everything before the {@code #} that separates it + * from the JSON Pointer. Empty for a reference that stays inside its own document, and null + * when there is no separator at all, in which case {@code messages} says so. + * + * This only reads the text. Turning what it gives back into somewhere a document can be + * retrieved from is {@link #resolveDocumentLocation}'s job. + */ + private static String extractLocationPart(String ref, List messages) { + + if (!ref.contains(FRAGMENT_SEPARATOR)) { + messages.add("Not a valid $ref, as it contains no " + FRAGMENT_SEPARATOR + ": " + ref); + return null; + } + + return ref.substring(0, ref.indexOf(FRAGMENT_SEPARATOR)); } } diff --git a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiAccessTest.java b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiAccessTest.java index 6fa08e8399..1c454bb2da 100644 --- a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiAccessTest.java +++ b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiAccessTest.java @@ -82,4 +82,93 @@ public void testAResourceThatIsNotThere() { assertTrue(e.getMessage().contains("classpath"), e.getMessage()); } + @Test + public void testAnotherDocumentNextToThisOneIsFollowed(@TempDir Path dir) throws IOException { + + write(dir.resolve("shared.yaml"), + "components:\n" + + " schemas:\n" + + " Thing:\n" + + " type: object\n" + + " properties:\n" + + " id:\n" + + " type: string\n"); + + Path main = write(dir.resolve("main.yaml"), + "asyncapi: 3.0.0\n" + + "info:\n" + + " title: Split across files\n" + + " version: 1.0.0\n" + + "components:\n" + + " messages:\n" + + " m:\n" + + " payload:\n" + + " $ref: 'shared.yaml#/components/schemas/Thing'\n"); + + AsyncApiDocument document = AsyncApiAccess.getAsyncApiFromLocation(main.toString()); + + assertTrue(document.getWarnings().isEmpty(), "unexpected warnings: " + document.getWarnings()); + assertEquals(1, document.getComponentSchemas().size()); + assertTrue(document.getComponentSchemas().keySet().iterator().next().startsWith("_ext_")); + assertTrue(document.getMessages().containsKey("m")); + } + + @Test + public void testAnotherDocumentIsFollowedFromAFolderWhoseNameHasASpace(@TempDir Path dir) + throws IOException { + + /* + A space is not legal in a URI, and resolving a relative reference used to go through + java.net.URI. That made following a reference depend on where the project happened + to sit on disk: every external reference was dropped, with a warning saying the file + did not exist, for any path containing a space -- and likewise on Windows, whose + paths have backslashes and a drive letter. + */ + Path folder = Files.createDirectories(dir.resolve("with space")); + + write(folder.resolve("shared.yaml"), + "components:\n" + + " schemas:\n" + + " Thing:\n" + + " type: string\n"); + + Path main = write(folder.resolve("main.yaml"), + "asyncapi: 3.0.0\n" + + "info:\n" + + " title: In a folder with a space in its name\n" + + " version: 1.0.0\n" + + "components:\n" + + " messages:\n" + + " m:\n" + + " payload:\n" + + " $ref: 'shared.yaml#/components/schemas/Thing'\n"); + + AsyncApiDocument document = AsyncApiAccess.getAsyncApiFromLocation(main.toString()); + + assertTrue(document.getWarnings().isEmpty(), "unexpected warnings: " + document.getWarnings()); + assertEquals(1, document.getComponentSchemas().size()); + assertTrue(document.getMessages().containsKey("m")); + } + + @Test + public void testTheSameLocationAlwaysGetsTheSameImportedNames(@TempDir Path dir) throws IOException { + + //the names imported components get have to be stable, or generated output would churn + write(dir.resolve("shared.yaml"), "components:\n schemas:\n Thing:\n type: string\n"); + + Path main = write(dir.resolve("main.yaml"), + "asyncapi: 3.0.0\n" + + "info:\n" + + " title: Split\n" + + " version: 1.0.0\n" + + "components:\n" + + " messages:\n" + + " m:\n" + + " payload:\n" + + " $ref: 'shared.yaml#/components/schemas/Thing'\n"); + + assertEquals( + AsyncApiAccess.getAsyncApiFromLocation(main.toString()).getComponentSchemas().keySet(), + AsyncApiAccess.getAsyncApiFromLocation(main.toString()).getComponentSchemas().keySet()); + } } diff --git a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiParserTest.java b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiParserTest.java index 582ea60c50..069cacc981 100644 --- a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiParserTest.java +++ b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiParserTest.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.regex.Pattern; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -815,4 +816,174 @@ public void testResolvingAnOperationThatDeclaresNoReply() { assertTrue(document.replyMessagesOf(operation).isEmpty()); } + // ------------------------------------------------------------------ other documents + + @Test + public void testSchemasFromAnotherDocumentAreInlined() { + + AsyncApiDocument document = load("/asyncapi/artificial/external-main.yaml"); + + assertTrue(document.getWarnings().isEmpty(), "unexpected warnings: " + document.getWarnings()); + + List imported = new ArrayList<>(); + for (String key : document.getComponentSchemas().keySet()) { + if (key.startsWith("_ext_")) { + imported.add(key); + } + } + assertEquals(2, imported.size(), "expected Order and Item to be imported, got " + imported); + + //the local schema of the same name must not have been overwritten + assertTrue(document.getComponentSchemas().containsKey("Order")); + assertTrue(document.getComponentSchemas().get("Order").get("properties").has("localOnly")); + + //the payload now points at the imported copy, not at the local schema of the same name + String payloadRef = document.getMessages().get("placeOrder").getPayload().get("$ref").asText(); + assertTrue(payloadRef.startsWith("#/components/schemas/_ext_"), payloadRef); + assertTrue(document.getComponentSchemas().containsKey( + payloadRef.substring("#/components/schemas/".length()))); + } + + @Test + public void testReferencesInsideAnImportedDocumentAreRewritten() { + + AsyncApiDocument document = load("/asyncapi/artificial/external-main.yaml"); + + Pattern expected = Pattern.compile("^_ext_[0-9a-f]{8}_Order$"); + String importedOrderKey = null; + + for (String key : document.getComponentSchemas().keySet()) { + if (expected.matcher(key).matches()) { + importedOrderKey = key; + break; + } + } + + assertNotNull(importedOrderKey, document.getComponentSchemas().keySet().toString()); + + String itemRef = document.getComponentSchemas().get(importedOrderKey) + .get("properties").get("item").get("$ref").asText(); + + //'#/components/schemas/Item' inside the other document means *its* Item + String prefix = importedOrderKey.substring(0, importedOrderKey.length() - "Order".length()); + assertEquals("#/components/schemas/" + prefix + "Item", itemRef); + assertTrue(document.getComponentSchemas().containsKey( + itemRef.substring("#/components/schemas/".length()))); + } + + @Test + public void testMessagesFromAnotherDocumentAreInlined() { + + AsyncApiMessage imported = + load("/asyncapi/artificial/external-main.yaml").getMessages().get("imported"); + + assertEquals("Ack", imported.getName()); + assertTrue(imported.getPayload().get("$ref").asText().startsWith("#/components/schemas/_ext_")); + } + + @Test + public void testPointerDeeperThanAnImportedSchemaKeepsItsTail() { + + AsyncApiDocument document = load("/asyncapi/artificial/external-main.yaml"); + + /* + Only the component key is renamed. Everything past it describes a way into the + imported schema and has to survive, or the reference would name the primary + document's own 'Order' -- a different schema that happens to share the name. + */ + String ref = document.getMessages().get("deepPointer").getPayload().get("$ref").asText(); + + assertTrue(ref.matches("^#/components/schemas/_ext_[0-9a-f]{8}_Order/properties/item$"), ref); + } + + @Test + public void testDeepLocalPointerInsideAnImportedDocumentIsRewritten() { + + AsyncApiDocument document = load("/asyncapi/artificial/external-main.yaml"); + + //the imported document wrote '#/components/schemas/Order/...' meaning its own Order + String ref = document.getMessages().get("importedDeep").getPayload().get("$ref").asText(); + + assertTrue(ref.matches("^#/components/schemas/_ext_[0-9a-f]{8}_Order/properties/id$"), ref); + + //and what it now names is the imported copy, which has the fields the other document declared + String key = ref.substring("#/components/schemas/".length(), ref.indexOf("/properties/")); + assertTrue(document.getComponentSchemas().get(key).get("properties").has("id")); + assertFalse(document.getComponentSchemas().get(key).get("properties").has("localOnly")); + } + + @Test + public void testReferenceIntoANonComponentPartOfAnotherDocumentIsRejected() { + + AsyncApiDocument document = load("/asyncapi/artificial/external-deep.yaml"); + + //what could be imported was, and the message using it is fine + assertTrue(document.getMessages().containsKey("placed")); + + //the other one points into an extension section, which cannot be brought in. Leaving it + //with a reference out of the document would only fail later, so it is dropped now + assertFalse(document.getMessages().containsKey("shipped")); + assertEquals(Arrays.asList("placed"), document.getChannels().get("shipments").getMessageIds()); + + //both halves are reported: what could not be imported, and what that cost + assertTrue(warns(document, "x-webhooks", "can be imported"), document.getWarnings().toString()); + assertTrue(warns(document, "message 'shipped'", "is ignored"), document.getWarnings().toString()); + } + + @Test + public void testAReferenceIsResolvedAgainstTheDocumentThatMakesIt() { + + AsyncApiDocument document = load("/asyncapi/artificial/nested/main.yaml"); + + /* + 'sub/nested.yaml' refers to 'shared.yaml', which for it means 'sub/shared.yaml' and + does not exist. Resolving that against the primary document's directory instead + would silently bind it to the shared.yaml next to main.yaml -- a different schema, + with no sign that anything went wrong. + */ + assertTrue( + warns(document, "Failed to retrieve", "sub/shared.yaml"), + document.getWarnings().toString()); + + //what could be imported was + assertTrue(document.getMessages().containsKey("fromShared")); + String sharedKey = document.getMessages().get("fromShared").getPayload().get("$ref").asText() + .substring("#/components/schemas/".length()); + assertEquals( + "the-one-next-to-main", + document.getComponentSchemas().get(sharedKey).get("title").asText()); + + //and the one that could not be is dropped rather than bound to the wrong schema + assertFalse(document.getMessages().containsKey("fromNested")); + assertEquals(Arrays.asList("fromShared"), document.getChannels().get("c").getMessageIds()); + } + + @Test + public void testADocumentThatNamesItselfIsNotImportedIntoItself() { + + AsyncApiDocument document = load("/asyncapi/artificial/self-reference.yaml"); + + assertTrue(document.getWarnings().isEmpty(), "unexpected warnings: " + document.getWarnings()); + + //one copy of everything, not two + assertEquals(setOf("Thing"), document.getComponentSchemas().keySet()); + assertEquals(setOf("m"), document.getMessages().keySet()); + + //and the long-winded reference now reads as the local one it always was + assertEquals( + "#/components/schemas/Thing", + document.getMessages().get("m").getPayload().get("$ref").asText()); + } + + @Test + public void testTextOnlyDocumentReportsUnresolvableExternalReferences() { + + AsyncApiDocument document = parse( + AsyncApiAccess.readFromResource("/asyncapi/artificial/external-main.yaml")); + + assertTrue( + warns(document, "supplied as text"), + "expected a warning about references that cannot be resolved: " + document.getWarnings()); + } + } diff --git a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiRefResolverTest.java b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiRefResolverTest.java index f7fea95c28..283f2b904a 100644 --- a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiRefResolverTest.java +++ b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/AsyncApiRefResolverTest.java @@ -2,10 +2,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.webfuzzing.asyncapi.mapper.AsyncApiMapper; +import com.webfuzzing.asyncapi.models.AsyncApiDocument; +import com.webfuzzing.asyncapi.models.DocumentLocation; +import com.webfuzzing.asyncapi.models.DocumentLocationType; +import com.webfuzzing.asyncapi.parser.AsyncApiParser; +import com.webfuzzing.asyncapi.resolver.AsyncApiDocumentFetcher; import com.webfuzzing.asyncapi.resolver.AsyncApiRefResolver; +import com.webfuzzing.asyncapi.resolver.RefLocations; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -13,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -40,7 +48,9 @@ private JsonNode document() throws IOException { + " with/slash:\n" + " type: string\n" + " with~tilde:\n" - + " type: integer\n"); + + " type: integer\n" + + " bad%zz:\n" + + " type: boolean\n"); } @Test @@ -81,6 +91,21 @@ public void testEscapedPointerSegments() throws IOException { .resolveLocal(document(), "#/components/schemas/with~0tilde").get("type").asText()); } + @Test + public void testPercentEncodedPointerSegmentsAreDecoded() throws IOException { + + //references are URIs, so a key may arrive percent-encoded as well as pointer-escaped + assertEquals("string", AsyncApiRefResolver + .resolveLocal(document(), "#/components/schemas/with%2Fslash").get("type").asText()); + + /* + "%zz" is not a valid escape. Rather than failing, the segment is taken as written -- + and here that is a key which exists, so the fallback is observable. + */ + assertEquals("boolean", AsyncApiRefResolver + .resolveLocal(document(), "#/components/schemas/bad%zz").get("type").asText()); + } + @Test public void testKeyOfAReferenceWithTheExpectedShape() { @@ -139,4 +164,371 @@ public void testRefOfSomethingThatIsNotAReference() throws IOException { .get("payload").get("properties").get("item"))); } + // ------------------------------------------------------------------ where a reference leads + + @Test + public void testAnAbsoluteReferenceIsTakenAsItIs() { + + List messages = new ArrayList<>(); + + assertEquals( + "https://example.com/shared.yaml", + RefLocations.resolveDocumentLocation( + "https://example.com/shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("/some/where/main.yaml"), + messages)); + + //plain http is absolute just the same + assertEquals( + "http://example.com/shared.yaml", + RefLocations.resolveDocumentLocation( + "http://example.com/shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("/some/where/main.yaml"), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testARelativeReferenceIsResolvedAgainstTheReferringDocument() { + + List messages = new ArrayList<>(); + + assertEquals( + "/some/where/shared.yaml", + RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("/some/where/main.yaml"), + messages)); + + //a document one directory down means that directory, not the primary document's + assertEquals( + "/some/where/sub/shared.yaml", + RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("/some/where/sub/nested.yaml"), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testDotAndDotDotSegmentsAreCollapsed() { + + List messages = new ArrayList<>(); + + /* + The resolved location is what tells imported documents apart, so two references to + the same document must come out as the same string however they were written. A + plain path is resolved through the file system and a URL per RFC 3986; both collapse + these segments. + */ + assertEquals( + "/a/common/x.yaml", + RefLocations.resolveDocumentLocation( + "../common/x.yaml#/components/schemas/X", + DocumentLocation.ofLocal("/a/b/main.yaml"), + messages)); + assertEquals( + "/a/b/shared.yaml", + RefLocations.resolveDocumentLocation( + "./shared.yaml#/components/schemas/X", + DocumentLocation.ofLocal("/a/b/main.yaml"), + messages)); + assertEquals( + "https://example.com/common/x.yaml", + RefLocations.resolveDocumentLocation( + "../common/x.yaml#/components/schemas/X", + DocumentLocation.ofRemote("https://example.com/a/main.yaml"), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testAPathWithASpaceIsNotAUriAndIsResolvedAsAPath() { + + List messages = new ArrayList<>(); + + //java.net.URI would reject this; a path is resolved through the file system instead + assertEquals( + "/has space/shared.yaml", + RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("/has space/main.yaml"), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testADocumentServedFromADirectoryUrl() { + + List messages = new ArrayList<>(); + + //a trailing slash means the URL is the folder itself, so nothing is stripped from it + assertEquals( + "https://example.com/docs/shared.yaml", + RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.ofRemote("https://example.com/docs/"), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testAFileUrlIsResolvedAsAUrl() { + + List messages = new ArrayList<>(); + + String resolved = RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("file:///a/b/main.yaml"), + messages); + + //URI is free to write one slash or three after "file:"; both name the same file + assertTrue(resolved.startsWith("file:"), resolved); + assertTrue(resolved.endsWith("/a/b/shared.yaml"), resolved); + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testAReferringUrlThatIsNotAValidUriIsReported() { + + List messages = new ArrayList<>(); + + //a URL is a URI, and one with a space in it is not one; there is no right answer here + assertNull(RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.ofRemote("https://example.com/has space/main.yaml"), + messages)); + + assertEquals(1, messages.size()); + assertTrue(messages.get(0).contains("not a valid URI"), messages.toString()); + } + + @Test + public void testAProtocolRelativeReferenceBorrowsTheProtocol() { + + List messages = new ArrayList<>(); + + assertEquals( + "https://other.com/shared.yaml", + RefLocations.resolveDocumentLocation( + "//other.com/shared.yaml#/components/schemas/Thing", + DocumentLocation.ofRemote("https://example.com/main.yaml"), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + @Test + public void testAProtocolRelativeReferenceWithNoProtocolToBorrow() { + + List messages = new ArrayList<>(); + + /* + A plain file path has no protocol, so there is nothing to borrow. Reporting it is + the only sensible answer -- taking the text apart regardless would read past the + start of the string. + */ + assertNull(RefLocations.resolveDocumentLocation( + "//other.com/shared.yaml#/components/schemas/Thing", + DocumentLocation.ofLocal("/some/where/main.yaml"), + messages)); + + assertEquals(1, messages.size()); + assertTrue(messages.get(0).contains("No protocol"), messages.toString()); + } + + @Test + public void testAReferenceWithNoFragmentIsNotOne() { + + List messages = new ArrayList<>(); + + assertNull(RefLocations.resolveDocumentLocation( + "shared.yaml", DocumentLocation.ofLocal("/some/where/main.yaml"), messages)); + + assertEquals(1, messages.size()); + assertTrue(messages.get(0).contains("contains no #"), messages.toString()); + } + + @Test + public void testARelativeReferenceFromADocumentWithNoLocation() { + + //a document handed over as text has no neighbours for a relative reference to name + assertThrows(IllegalArgumentException.class, () -> RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + DocumentLocation.MEMORY, + new ArrayList())); + } + + @Test + public void testALocalReferenceIsRecognisedWhateverItPointsAt() { + + assertTrue(RefLocations.isLocalRef("#/components/schemas/Thing")); + assertTrue(RefLocations.isLocalRef("#")); + assertFalse(RefLocations.isLocalRef("shared.yaml#/components/schemas/Thing")); + assertFalse(RefLocations.isLocalRef("https://example.com/shared.yaml#/x")); + } + + @Test + public void testWhereADocumentReadFromTheClasspathLooksForItsNeighbours() { + + List messages = new ArrayList<>(); + + //a resource path is resolved the same way a file path is + assertEquals( + "/asyncapi/artificial/shared.yaml", + RefLocations.resolveDocumentLocation( + "shared.yaml#/components/schemas/Thing", + new DocumentLocation("/asyncapi/artificial/main.yaml", DocumentLocationType.RESOURCE), + messages)); + + assertTrue(messages.isEmpty(), messages.toString()); + } + + // ------------------------------------------------------------------ importing other documents + + private static final String SHARED_THING = + "components:\n" + + " schemas:\n" + + " Thing:\n" + + " type: string\n"; + + private static String mainReferring(String ref) { + return "asyncapi: 3.0.0\n" + + "info:\n" + + " title: Split across documents\n" + + " version: 1.0.0\n" + + "components:\n" + + " messages:\n" + + " m:\n" + + " payload:\n" + + " $ref: '" + ref + "'\n"; + } + + private static int importedSchemas(AsyncApiDocument document) { + + int count = 0; + + for (String key : document.getComponentSchemas().keySet()) { + if (key.startsWith("_ext_")) { + count++; + } + } + + return count; + } + + private boolean warns(AsyncApiDocument document, String text) { + + for (String warning : document.getWarnings()) { + if (warning.contains(text)) { + return true; + } + } + + return false; + } + + @Test + public void testAnotherDocumentIsFetchedFromARemoteLocation() { + + List fetched = new ArrayList<>(); + List fetchedAs = new ArrayList<>(); + + /* + The fetcher is an interface precisely so that this can be tested without a server: + it is handed the absolute location and told how to read it, and hands back the text. + */ + AsyncApiDocumentFetcher fetcher = (location, type) -> { + fetched.add(location); + fetchedAs.add(type); + return SHARED_THING; + }; + + AsyncApiDocument document = AsyncApiParser.parse( + mainReferring("shared.yaml#/components/schemas/Thing"), + DocumentLocation.ofRemote("https://example.com/api/main.yaml"), + fetcher); + + //resolved against the referring URL, and read the way a URL is read + assertEquals(Arrays.asList("https://example.com/api/shared.yaml"), fetched); + assertEquals(Arrays.asList(DocumentLocationType.REMOTE), fetchedAs); + + assertTrue(document.getWarnings().isEmpty(), document.getWarnings().toString()); + assertEquals(1, importedSchemas(document)); + assertTrue(document.getMessages().containsKey("m")); + } + + @Test + public void testNoMoreThanAHundredDocumentsAreImported() { + + /* + References are paths, and a server that answers every path -- or a symlink loop -- + would be followed for ever without a ceiling. Here every location resolves to a + document, and the message points at one more of them than the ceiling allows. + */ + StringBuilder main = new StringBuilder( + "asyncapi: 3.0.0\n" + + "info:\n" + + " title: Too many neighbours\n" + + " version: 1.0.0\n" + + "components:\n" + + " messages:\n" + + " m:\n" + + " payload:\n" + + " type: object\n" + + " properties:\n"); + + for (int i = 0; i <= 100; i++) { + main.append(" p").append(i).append(":\n") + .append(" $ref: 'doc").append(i).append(".yaml#/components/schemas/Thing'\n"); + } + + AsyncApiDocument document = AsyncApiParser.parse( + main.toString(), + DocumentLocation.ofRemote("https://example.com/main.yaml"), + (location, type) -> SHARED_THING); + + assertEquals(100, importedSchemas(document)); + assertTrue(warns(document, "More than 100 documents"), document.getWarnings().toString()); + + //the reference past the ceiling was left unresolved, and the message depending on it goes + assertFalse(document.getMessages().containsKey("m")); + } + + @Test + public void testAReferenceToAWholeDocumentIsReported() { + + //nothing after the '#': it names the other document itself rather than a component of it + AsyncApiDocument document = AsyncApiParser.parse( + mainReferring("shared.yaml#"), + DocumentLocation.ofRemote("https://example.com/main.yaml"), + (location, type) -> SHARED_THING); + + assertTrue(warns(document, "whole document"), document.getWarnings().toString()); + assertFalse(document.getMessages().containsKey("m")); + } + + @Test + public void testAReferenceWithNoFragmentAtAllIsReportedAndNotFetched() { + + List fetched = new ArrayList<>(); + + //not a $ref at all by the specification's definition, so there is nothing to retrieve + AsyncApiDocument document = AsyncApiParser.parse( + mainReferring("shared.yaml"), + DocumentLocation.ofRemote("https://example.com/main.yaml"), + (location, type) -> { + fetched.add(location); + return SHARED_THING; + }); + + assertTrue(fetched.isEmpty(), "nothing should have been fetched, but got " + fetched); + assertTrue(warns(document, "contains no #"), document.getWarnings().toString()); + assertFalse(document.getMessages().containsKey("m")); + } } diff --git a/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/DocumentLocationTest.java b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/DocumentLocationTest.java new file mode 100644 index 0000000000..ab919046d3 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/java/com/webfuzzing/asyncapi/DocumentLocationTest.java @@ -0,0 +1,53 @@ +package com.webfuzzing.asyncapi; + +import com.webfuzzing.asyncapi.models.DocumentLocation; +import com.webfuzzing.asyncapi.models.DocumentLocationType; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DocumentLocationTest { + + @Test + public void testEqualityIsByLocationAndType() { + + DocumentLocation a = DocumentLocation.ofLocal("/a/main.yaml"); + DocumentLocation same = new DocumentLocation("/a/main.yaml", DocumentLocationType.LOCAL); + + assertEquals(a, a); + assertEquals(a, same); + assertEquals(a.hashCode(), same.hashCode()); + + //the same text read a different way is a different location + assertNotEquals(a, new DocumentLocation("/a/main.yaml", DocumentLocationType.RESOURCE)); + assertNotEquals(a, DocumentLocation.ofLocal("/b/main.yaml")); + + assertNotEquals(a, null); + assertNotEquals(a, "/a/main.yaml"); + } + + @Test + public void testToStringSaysWhatAndWhere() { + + String text = DocumentLocation.ofRemote("https://example.com/main.yaml").toString(); + + assertTrue(text.contains("REMOTE"), text); + assertTrue(text.contains("https://example.com/main.yaml"), text); + } + + @Test + public void testAPlainPathIsOnlyALocalLocationWithoutAScheme() { + + assertTrue(DocumentLocation.ofLocal("/a/main.yaml").isPlainFilePath()); + assertTrue(DocumentLocation.ofLocal("C:\\dir\\main.yaml").isPlainFilePath()); + + assertFalse(DocumentLocation.ofLocal("file:///a/main.yaml").isPlainFilePath()); + assertFalse(DocumentLocation.ofLocal("FILE:///a/main.yaml").isPlainFilePath()); + assertFalse(DocumentLocation.ofRemote("https://example.com/main.yaml").isPlainFilePath()); + assertFalse(DocumentLocation.ofResource("/asyncapi/main.yaml").isPlainFilePath()); + assertFalse(DocumentLocation.MEMORY.isPlainFilePath()); + } +} diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-deep.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-deep.yaml new file mode 100644 index 0000000000..3b29912219 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-deep.yaml @@ -0,0 +1,31 @@ +asyncapi: 3.0.0 +info: + title: A reference into a part of another document that cannot be imported + version: 1.0.0 + +channels: + shipments: + address: shipments + messages: + shipped: + $ref: '#/components/messages/shipped' + placed: + $ref: '#/components/messages/placed' + +operations: + onShipment: + action: receive + channel: + $ref: '#/channels/shipments' + +components: + messages: + shipped: + name: OrderShipped + # not a component of the other document, so there is nothing to bring in + payload: + $ref: 'external-shared.yaml#/x-webhooks/orderShipped/schema' + placed: + name: OrderPlaced + payload: + $ref: 'external-shared.yaml#/components/schemas/Order' diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-main.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-main.yaml new file mode 100644 index 0000000000..81137aba3c --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-main.yaml @@ -0,0 +1,52 @@ +asyncapi: 3.0.0 +info: + title: References into another document + version: 1.0.0 + +servers: + broker: + host: localhost:9092 + protocol: kafka + +channels: + orders: + address: orders.request + messages: + placeOrder: + $ref: '#/components/messages/placeOrder' + +operations: + placeOrder: + action: receive + channel: + $ref: '#/channels/orders' + +components: + messages: + placeOrder: + name: PlaceOrder + contentType: application/json + payload: + $ref: 'external-shared.yaml#/components/schemas/Order' + + # a message living in the other document as well + imported: + $ref: 'external-shared.yaml#/components/messages/Ack' + + # a pointer into the other document that goes deeper than the schema it names + deepPointer: + payload: + $ref: 'external-shared.yaml#/components/schemas/Order/properties/item' + + # and the same, written by the other document about itself + importedDeep: + $ref: 'external-shared.yaml#/components/messages/DeepAck' + + schemas: + # a schema of our own, whose name deliberately collides with one in the other document + Order: + type: object + required: [localOnly] + properties: + localOnly: + type: boolean diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-shared.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-shared.yaml new file mode 100644 index 0000000000..d675010be0 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/external-shared.yaml @@ -0,0 +1,45 @@ +openapi: 3.0.0 +info: + title: Shared schemas, not an AsyncAPI document itself + version: 1.0.0 + +# a schema living somewhere that is not a component, as OpenAPI extensions often do +x-webhooks: + orderShipped: + schema: + type: object + properties: + shippedAt: + type: string + +components: + messages: + Ack: + name: Ack + payload: + $ref: '#/components/schemas/Order' + + # a local pointer that goes deeper than the schema it names. It has to end up naming *this* + # document's Order, not the one of the same name in the document importing it + DeepAck: + name: DeepAck + payload: + $ref: '#/components/schemas/Order/properties/id' + + schemas: + Order: + type: object + required: [id, item] + properties: + id: + type: string + item: + $ref: '#/components/schemas/Item' + Item: + type: object + properties: + sku: + type: string + quantity: + type: integer + minimum: 1 diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/main.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/main.yaml new file mode 100644 index 0000000000..ea4d4c3c63 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/main.yaml @@ -0,0 +1,35 @@ +asyncapi: 3.0.0 +info: + title: Imports from two directories, one of which has a broken reference of its own + version: 1.0.0 + +# The point of this document is a trap: 'sub/nested.yaml' refers to 'shared.yaml', which for it +# means 'sub/shared.yaml' — a file that does not exist. Resolving that reference against the +# wrong directory would bind it to the 'shared.yaml' next to *this* file instead, which is a +# different schema entirely, and nothing would say so. + +channels: + c: + address: c + messages: + fromShared: + $ref: '#/components/messages/fromShared' + fromNested: + $ref: '#/components/messages/fromNested' + +operations: + onC: + action: receive + channel: + $ref: '#/channels/c' + +components: + messages: + fromShared: + name: FromShared + payload: + $ref: 'shared.yaml#/components/schemas/Thing' + fromNested: + name: FromNested + payload: + $ref: 'sub/nested.yaml#/components/schemas/Thing' diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/shared.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/shared.yaml new file mode 100644 index 0000000000..5aba0d9892 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/shared.yaml @@ -0,0 +1,13 @@ +asyncapi: 3.0.0 +info: + title: The shared document next to main.yaml + version: 1.0.0 + +components: + schemas: + Thing: + title: the-one-next-to-main + type: object + properties: + id: + type: string diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/sub/nested.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/sub/nested.yaml new file mode 100644 index 0000000000..ad05ff7939 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/nested/sub/nested.yaml @@ -0,0 +1,14 @@ +asyncapi: 3.0.0 +info: + title: A document one directory down + version: 1.0.0 + +components: + schemas: + Thing: + title: the-one-in-sub + type: object + properties: + # for this document, 'shared.yaml' means sub/shared.yaml, which does not exist + borrowed: + $ref: 'shared.yaml#/components/schemas/Thing' diff --git a/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/self-reference.yaml b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/self-reference.yaml new file mode 100644 index 0000000000..689a554875 --- /dev/null +++ b/core-extra/asyncapi-parser/src/test/resources/asyncapi/artificial/self-reference.yaml @@ -0,0 +1,35 @@ +asyncapi: 3.0.0 +info: + title: A document that refers to itself by name + version: 1.0.0 + +# Some generators write every reference as an absolute one, including those that stay inside +# the document. Following that as if it were another document would import the whole file into +# itself, silently doubling its schemas and messages. + +channels: + c: + address: c + messages: + m: + $ref: '#/components/messages/m' + +operations: + onC: + action: receive + channel: + $ref: '#/channels/c' + +components: + messages: + m: + name: M + payload: + $ref: 'self-reference.yaml#/components/schemas/Thing' + + schemas: + Thing: + type: object + properties: + id: + type: string