diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java new file mode 100644 index 0000000000..1ab520f5fb --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.StringUtil; + +/** + * One labeled edge of a {@link DependencyGraph}: the token at {@code dependent} is governed + * by the token at {@code head} under the given {@code relation}. + * + *

Indices are zero-based positions in the token array the graph was built over. A + * {@code head} of {@link #ROOT_HEAD} marks the dependent as the sentence root, which is + * attached to the artificial root node rather than to another token.

+ * + * @param head The zero-based index of the governing token, or {@link #ROOT_HEAD} when the + * dependent is the sentence root. + * @param dependent The zero-based index of the governed token. + * @param relation The dependency relation label, for example {@code nsubj}. + * + * @since 3.0.0 + */ +public record DependencyArc(int head, int dependent, String relation) { + + /** + * The {@code head} value marking an arc from the artificial root node. + */ + public static final int ROOT_HEAD = -1; + + /** + * Validates the arc invariants. + * + * @throws IllegalArgumentException Thrown if {@code dependent} is negative, {@code head} + * is less than {@link #ROOT_HEAD}, the arc is a self-loop, or {@code relation} + * is {@code null} or blank. + */ + public DependencyArc { + if (dependent < 0) { + throw new IllegalArgumentException("dependent must not be negative: " + dependent); + } + if (head < ROOT_HEAD) { + throw new IllegalArgumentException("head must be a token index or ROOT_HEAD: " + head); + } + if (head == dependent) { + throw new IllegalArgumentException("arc must not be a self-loop: " + head); + } + if (relation == null || StringUtil.isBlank(relation)) { + throw new IllegalArgumentException("relation must not be null or blank"); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java new file mode 100644 index 0000000000..40a1f9c90b --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import opennlp.tools.util.StringUtil; + +/** + * An immutable dependency tree over one sentence: for every token, the index of its head + * and the label of the relation to that head. + * + *

Token indices are zero-based positions in the sentence the graph was built for. + * Exactly one token carries the head value {@link DependencyArc#ROOT_HEAD}, marking it as + * the sentence root. Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public final class DependencyGraph { + + private final int[] heads; + private final String[] relations; + + /** + * Wraps already validated arrays; instances are created through {@link #of}. + */ + private DependencyGraph(int[] heads, String[] relations) { + this.heads = heads; + this.relations = relations; + } + + /** + * Creates a {@link DependencyGraph} from parallel head and relation arrays. + * + * @param heads For each token, the zero-based index of its head token, or + * {@link DependencyArc#ROOT_HEAD} for the sentence root. Must not be + * {@code null} or empty, every value must be a valid token index or + * {@link DependencyArc#ROOT_HEAD}, no token may head itself, and exactly + * one token must be the root. + * @param relations For each token, the label of the relation to its head. Must not be + * {@code null}, must have the same length as {@code heads}, and no + * entry may be {@code null} or blank. + * @return A validated {@link DependencyGraph}. Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + public static DependencyGraph of(int[] heads, String[] relations) { + if (heads == null || relations == null) { + throw new IllegalArgumentException("heads and relations must not be null"); + } + if (heads.length == 0) { + throw new IllegalArgumentException("a dependency graph needs at least one token"); + } + if (heads.length != relations.length) { + throw new IllegalArgumentException("heads and relations must have the same length: " + + heads.length + " != " + relations.length); + } + int roots = 0; + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + roots++; + } else if (heads[i] < 0 || heads[i] >= heads.length) { + throw new IllegalArgumentException("head of token " + i + + " is out of range: " + heads[i]); + } else if (heads[i] == i) { + throw new IllegalArgumentException("token " + i + " must not head itself"); + } + if (relations[i] == null || StringUtil.isBlank(relations[i])) { + throw new IllegalArgumentException("relation of token " + i + " must not be blank"); + } + } + if (roots != 1) { + throw new IllegalArgumentException("expected exactly one root, found " + roots); + } + return new DependencyGraph(heads.clone(), relations.clone()); + } + + /** + * @return The number of tokens the graph spans. + */ + public int size() { + return heads.length; + } + + /** + * Retrieves the head of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The zero-based index of the head token, or {@link DependencyArc#ROOT_HEAD} + * when the token is the sentence root. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int headOf(int index) { + checkIndex(index); + return heads[index]; + } + + /** + * Retrieves the relation label of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The label of the relation between the token and its head. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String relationOf(int index) { + checkIndex(index); + return relations[index]; + } + + /** + * @return The zero-based index of the sentence root token. + */ + public int root() { + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + return i; + } + } + throw new IllegalStateException("graph invariant violated: no root present"); + } + + /** + * @return All arcs of the graph in token order, one per token. Never {@code null}. + */ + public List arcs() { + final List arcs = new ArrayList<>(heads.length); + for (int i = 0; i < heads.length; i++) { + arcs.add(new DependencyArc(heads[i], i, relations[i])); + } + return Collections.unmodifiableList(arcs); + } + + /** + * Rejects a token index outside {@code [0, size())}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + private void checkIndex(int index) { + if (index < 0 || index >= heads.length) { + throw new IllegalArgumentException("token index out of range: " + index + + ", size: " + heads.length); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencyGraph other)) { + return false; + } + return Arrays.equals(heads, other.heads) && Arrays.equals(relations, other.relations); + } + + @Override + public int hashCode() { + return 31 * Arrays.hashCode(heads) + Arrays.hashCode(relations); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < heads.length; i++) { + if (i > 0) { + sb.append(' '); + } + sb.append(i).append("<-").append(heads[i]).append(':').append(relations[i]); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java new file mode 100644 index 0000000000..bb9b4029e3 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The interface for dependency parsers, which assign every token of a sentence a syntactic + * head and a relation label, forming a single-rooted tree over the sentence. + * + *

Dependency parsing complements the constituency {@link opennlp.tools.parser.Parser}: + * where a constituency parse groups tokens into nested phrases, a dependency parse links + * each token directly to the token it modifies. The result is a {@link DependencyGraph} + * whose indices refer back to the input token array, so spans computed for those tokens + * remain valid for the parse.

+ * + *

Thread safety is implementation specific.

+ * + * @see DependencyGraph + * @since 3.0.0 + */ +public interface DependencyParser { + + /** + * Parses a sentence into a {@link DependencyGraph}. + * + * @param tokens The tokens of one sentence. Must not be {@code null} and must contain + * at least one token. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must have the same length as {@code tokens}. + * @return A {@link DependencyGraph} over the given tokens. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokens} or {@code tags} is + * {@code null}, empty, or of mismatched length. + */ + DependencyGraph parse(String[] tokens, String[] tags); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java new file mode 100644 index 0000000000..b805e79a51 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.Arrays; +import java.util.Objects; + +/** + * One dependency-annotated sentence: tokens, their part-of-speech tags, and the gold + * {@link DependencyGraph} over them. Used for training and evaluating a + * {@link DependencyParser}. + * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public class DependencySample { + + private final String[] tokens; + private final String[] tags; + private final DependencyGraph graph; + + /** + * Initializes a {@link DependencySample}. + * + * @param tokens The tokens of the sentence. Must not be {@code null} or empty. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must have the same length as {@code tokens}. + * @param graph The dependency graph over the tokens. Must not be {@code null} and its + * {@link DependencyGraph#size()} must equal the number of tokens. + * @throws IllegalArgumentException Thrown if any parameter is {@code null} or the + * lengths disagree. + */ + public DependencySample(String[] tokens, String[] tags, DependencyGraph graph) { + if (tokens == null || tags == null || graph == null) { + throw new IllegalArgumentException("tokens, tags and graph must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("a sample needs at least one token"); + } + if (tokens.length != tags.length || tokens.length != graph.size()) { + throw new IllegalArgumentException("tokens, tags and graph must agree in length: " + + tokens.length + ", " + tags.length + ", " + graph.size()); + } + this.tokens = tokens.clone(); + this.tags = tags.clone(); + this.graph = graph; + } + + /** + * @return The tokens of the sentence. Never {@code null}. + */ + public String[] getTokens() { + return tokens.clone(); + } + + /** + * @return The part-of-speech tags aligned with the tokens. Never {@code null}. + */ + public String[] getTags() { + return tags.clone(); + } + + /** + * @return The dependency graph over the tokens. Never {@code null}. + */ + public DependencyGraph getGraph() { + return graph; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencySample other)) { + return false; + } + return Arrays.equals(tokens, other.tokens) && Arrays.equals(tags, other.tags) + && graph.equals(other.graph); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(tokens), Arrays.hashCode(tags), graph); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < tokens.length; i++) { + sb.append(i + 1).append('\t').append(tokens[i]).append('\t').append(tags[i]) + .append('\t').append(graph.headOf(i) + 1).append('\t').append(graph.relationOf(i)) + .append(System.lineSeparator()); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java new file mode 100644 index 0000000000..3d3aaa1fd0 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import opennlp.tools.util.Span; + +/** + * One annotation of a {@link Document}: a typed value anchored to a {@link Span} of the + * document's original text, or a span-less value under a + * {@link LayerKey.Scope#DOCUMENT document-scoped} key. + * + *

The span always refers to the text the document was created with, never to a + * normalized or otherwise derived form, so any annotation can be highlighted in what the + * caller supplied. Whether a span is present is decided by the layer key's scope, not + * per annotation: the container rejects a span-less annotation under a positional key + * and a spanned annotation under a document-scoped key. Annotations that need to + * reference other annotations, for example a dependency arc naming its head token, do + * so by the index of the target annotation within its layer, never by object + * identity.

+ * + * @param span The location of the annotation in the original text, or {@code null} for + * a value under a document-scoped key. + * @param value The annotation value. Must not be {@code null}. + * @param The type of the annotation value. + * + * @since 3.0.0 + */ +public record Annotation(Span span, T value) { + + /** + * Validates the annotation. + * + * @throws IllegalArgumentException Thrown if {@code value} is {@code null}. + */ + public Annotation { + if (value == null) { + throw new IllegalArgumentException("value must not be null"); + } + } + + /** + * Creates a span-less annotation for a {@link LayerKey.Scope#DOCUMENT + * document-scoped} layer. + * + * @param value The annotation value. Must not be {@code null}. + * @param The type of the annotation value. + * @return An {@link Annotation} without a span. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code value} is {@code null}. + */ + public static Annotation of(T value) { + return new Annotation<>(null, value); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java new file mode 100644 index 0000000000..4d22d3a18b --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +/** + * An offset-anchored annotation container: the original text of one document plus any + * number of typed annotation layers over it. + * + *

A layer is a list of {@link Annotation annotations} identified by a + * {@link LayerKey}. The container itself knows nothing about specific layers; every + * analysis capability contributes its results as one more layer without any change to + * this interface, which is what keeps new capabilities additive. All spans refer to + * {@link #text()} as supplied, never to a derived form. A + * {@link LayerKey.Scope#DOCUMENT document-scoped} layer carries whole-document values + * without spans, for example a language id.

+ * + *

A document is never modified in place: {@link #with(LayerKey, List)} leaves its + * receiver untouched and returns a new document. Thread safety is implementation + * specific.

+ * + *

Three invariants make index-based references sound. A layer preserves its + * insertion order, and the container never sorts or reorders it. A layer is immutable + * once added: the returned lists reject modification and are detached from the + * caller's input list. Providing a layer that already exists is rejected loudly: the + * add is once-only, and the exception names the offending key. An annotation that + * references another annotation by its index within a layer, for example a dependency + * arc naming its head token, therefore stays valid for the lifetime of the + * document.

+ * + * @since 3.0.0 + */ +public interface Document { + + /** + * Creates an empty {@link Document} over a text. The returned document is immutable + * and safe to share between threads: it captures the text's content at construction, + * so later changes to a mutable {@code CharSequence} do not reach the document. + * + * @param text The original document text. Must not be {@code null}. + * @return A {@link Document} without any layers. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static Document of(CharSequence text) { + return ImmutableDocument.empty(text); + } + + /** + * @return The original text of the document. Never {@code null}. + */ + CharSequence text(); + + /** + * Retrieves the annotations of one layer. + * + * @param layer The layer to read. Must not be {@code null}. + * @param The type of the layer's annotation values. + * @return The layer's annotations in their layer order, or an empty list when the + * layer is absent. Never {@code null}; the list is unmodifiable. + * @throws IllegalArgumentException Thrown if {@code layer} is {@code null}. + */ + List> get(LayerKey layer); + + /** + * @return The keys of all layers present on the document. Never {@code null}; the set + * is unmodifiable. + */ + Set> layers(); + + /** + * Returns a new document with one layer added. + * + * @param layer The key of the layer to add. Must not be {@code null} and must not + * already be present. + * @param annotations The annotations of the layer. Must not be {@code null}, must not + * contain {@code null}, and every value must be assignable to the + * layer's type. Under a positional key every annotation must carry + * a span within the text bounds; under a document-scoped key no + * annotation may carry a span. + * @param The type of the layer's annotation values. + * @return A new {@link Document} sharing this document's text and existing layers. + * Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + Document with(LayerKey layer, List> annotations); + + /** + * How {@link #merge(Document, DuplicateLayerPolicy)} treats a layer key that is + * present on both documents. + */ + enum DuplicateLayerPolicy { + + /** Reject any layer key present on both documents. */ + REJECT, + + /** + * Keep one copy of a layer key present on both documents when the two layers are + * structurally equal, for example when two parallel branches ran the same + * tokenizer. Layers whose contents differ are rejected as with {@link #REJECT}. + * Equality is {@link Annotation} equality: spans compare by offsets and type, + * never by probability, and values by their own {@code equals}. + */ + KEEP_EQUAL + } + + /** + * Returns a new document combining this document's layers with another document's + * layers over the same text, joining documents grown independently, for example by + * pipelines that ran in parallel. + * + * @param other The document whose layers are added on top of this document's layers. + * Must not be {@code null}, must carry the same text content, and must + * not provide a layer this document already has. + * @return A new {@link Document} carrying the layers of both documents. Never + * {@code null}; both source documents are left untouched. + * @throws IllegalArgumentException Thrown if {@code other} is {@code null}, if its + * text content differs, or if a layer key is present on both documents; the + * exception names the offending key. + */ + default Document merge(Document other) { + return merge(other, DuplicateLayerPolicy.REJECT); + } + + /** + * Returns a new document combining this document's layers with another document's + * layers over the same text, resolving duplicate layer keys with + * {@code duplicateLayers}. + * + * @param other The document whose layers are added on top of this document's layers. + * Must not be {@code null} and must carry the same text content. + * @param duplicateLayers How to treat a layer key present on both documents. Must + * not be {@code null}. + * @return A new {@link Document} carrying the layers of both documents. Never + * {@code null}; both source documents are left untouched. + * @throws IllegalArgumentException Thrown if either argument is {@code null}, if the + * text content differs, or if a layer key is present on both documents and + * the policy does not keep it; the exception names the offending key. + */ + default Document merge(Document other, DuplicateLayerPolicy duplicateLayers) { + if (other == null) { + throw new IllegalArgumentException("other must not be null"); + } + if (duplicateLayers == null) { + throw new IllegalArgumentException("duplicateLayers must not be null"); + } + if (!text().toString().contentEquals(other.text())) { + throw new IllegalArgumentException( + "merge requires both documents to carry the same text"); + } + Document merged = this; + for (final LayerKey layer : other.layers()) { + if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL + && merged.layers().contains(layer)) { + if (layersEqual(merged, layer, other)) { + continue; + } + throw new IllegalArgumentException( + "layer is present on both documents with differing contents: " + layer); + } + merged = addLayer(merged, layer, other); + } + return merged; + } + + /** + * @return Whether the two documents carry structurally equal contents for the layer. + */ + private static boolean layersEqual(Document first, LayerKey layer, Document second) { + return first.get(layer).equals(second.get(layer)); + } + + /** + * Adds one layer of {@code from} to {@code base} through {@link #with(LayerKey, List)}, + * capturing the key's value type. + */ + private static Document addLayer(Document base, LayerKey layer, Document from) { + return base.with(layer, from.get(layer)); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java new file mode 100644 index 0000000000..e12a468b5a --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Runs a fixed sequence of {@link DocumentAnnotator annotators} over a text, producing + * one {@link Document} that carries every step's layers. + * + *

The pipeline is validated at build time: every annotator's required layers must be + * provided by an earlier annotator, and no two annotators may provide the same layer, so + * a misordered or conflicting pipeline fails when it is assembled rather than midway + * through a document. The analyzer holds no per-call state; it is as thread-safe as the + * annotators it is built from.

+ * + * @since 3.0.0 + */ +public final class DocumentAnalyzer { + + private final List annotators; + + private DocumentAnalyzer(List annotators) { + this.annotators = annotators; + } + + /** + * @return A new {@link Builder}. Never {@code null}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Analyzes a text by running every annotator in order. + * + * @param text The original document text. Must not be {@code null}. + * @return The annotated {@link Document}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + public Document analyze(CharSequence text) { + Document document = Document.of(text); + for (final DocumentAnnotator annotator : annotators) { + document = annotator.annotate(document); + } + return document; + } + + /** + * Assembles a {@link DocumentAnalyzer} from annotators in execution order. + */ + public static final class Builder { + + private final List annotators = new ArrayList<>(); + + private Builder() { + } + + /** + * Appends an annotator to the pipeline. + * + * @param annotator The annotator to run after the ones already added. Must not be + * {@code null}. + * @return This {@link Builder}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code annotator} is {@code null}. + */ + public Builder add(DocumentAnnotator annotator) { + if (annotator == null) { + throw new IllegalArgumentException("annotator must not be null"); + } + annotators.add(annotator); + return this; + } + + /** + * Validates the pipeline and builds the analyzer. + * + * @return A {@link DocumentAnalyzer}. Never {@code null}. + * @throws IllegalArgumentException Thrown if the pipeline is empty, an annotator + * requires a layer no earlier annotator provides, or two annotators provide + * the same layer. + */ + public DocumentAnalyzer build() { + if (annotators.isEmpty()) { + throw new IllegalArgumentException("a pipeline needs at least one annotator"); + } + final Map, Integer> providers = new HashMap<>(); + for (int position = 0; position < annotators.size(); position++) { + final DocumentAnnotator annotator = annotators.get(position); + for (final LayerKey required : annotator.requires()) { + if (!providers.containsKey(required)) { + throw new IllegalArgumentException("annotator " + annotator + + " requires layer " + required + ", which no earlier annotator provides"); + } + } + for (final LayerKey provided : annotator.provides()) { + final Integer earlier = providers.putIfAbsent(provided, position); + if (earlier != null) { + throw new IllegalArgumentException("annotators at positions " + earlier + + " and " + position + " both provide layer " + provided); + } + } + } + return new DocumentAnalyzer(List.copyOf(annotators)); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java new file mode 100644 index 0000000000..6bc8417e6c --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Set; + +/** + * A pipeline step that reads layers from a {@link Document} and returns a new document + * with its own layers added. + * + *

An annotator declares the layers it {@link #requires()} and {@link #provides()}, so + * a {@link DocumentAnalyzer} can validate a pipeline before running it. Annotators are + * usually thin adapters over an existing analysis component. Thread safety is + * implementation specific.

+ * + * @since 3.0.0 + */ +public interface DocumentAnnotator { + + /** + * Annotates a document. + * + *

A required layer must be present on the document, but it may be empty: an empty + * required layer is valid input and yields the annotator's provided layers present but + * empty, so a pipeline degrades gracefully on documents without content.

+ * + * @param document The document to annotate. Must not be {@code null} and must contain + * every layer named by {@link #requires()}. + * @return A new {@link Document} carrying the layers named by {@link #provides()} in + * addition to the input layers. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or lacks + * a required layer. + */ + Document annotate(Document document); + + /** + * @return The keys of the layers this annotator reads. Never {@code null}. + */ + default Set> requires() { + return Set.of(); + } + + /** + * @return The keys of the layers this annotator adds. Never {@code null}. + */ + Set> provides(); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java new file mode 100644 index 0000000000..fd5f81b858 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +/** + * Support methods shared by {@link DocumentAnnotator} implementations: the + * required-layer check and the per-sentence walk over the token layer. + * + *

These helpers keep the annotators' shared behavior identical across + * implementations: an absent required layer is always rejected with the same message + * naming the layer, and every per-sentence adapter applies the same sentence-to-token + * mapping, including the loud rejection of a token lying outside every sentence.

+ * + * @since 3.0.0 + */ +public final class DocumentAnnotators { + + /** + * Receives one sentence's contiguous token run during + * {@link #forEachSentence(List, List, SentenceTokenConsumer)}. + */ + @FunctionalInterface + public interface SentenceTokenConsumer { + + /** + * Consumes one sentence's tokens. + * + * @param first The position of the sentence's first token in the token layer. + * @param words The sentence's token values in layer order. Never {@code null} or + * empty; the run covers the token layer positions + * {@code [first, first + words.length)}. + */ + void accept(int first, String[] words); + } + + /** + * Verifies that a document is present and carries every given layer. + * + * @param document The document to check. + * @param layers The required layers, in the order they are to be reported. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, or if + * a layer is absent; the message names the first absent layer. + */ + public static void requireLayers(Document document, LayerKey... layers) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final Set> present = document.layers(); + for (final LayerKey layer : layers) { + if (!present.contains(layer)) { + throw new IllegalArgumentException("document lacks the required layer " + layer); + } + } + } + + /** + * Walks the token layer sentence by sentence and hands each sentence's contiguous + * token run to the consumer. + * + *

Both layers must be in text order. Each sentence consumes the contiguous run of + * tokens whose spans it encloses; a sentence without tokens is skipped. Every token + * must belong to a sentence: a token lying outside every sentence is rejected loudly + * after the walk, so it can never be silently dropped.

+ * + * @param sentences The sentence layer, in text order. Must not be {@code null}. + * @param tokens The token layer, in text order. Must not be {@code null}. + * @param consumer Receives each token-carrying sentence's run. Must not be + * {@code null}. + * @throws IllegalArgumentException Thrown if a token lies outside every sentence. + */ + public static void forEachSentence(List> sentences, + List> tokens, SentenceTokenConsumer consumer) { + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + } + consumer.accept(first, words); + } + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); + } + } + + private DocumentAnnotators() { + // Not instantiated; this class provides static support methods only. + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java new file mode 100644 index 0000000000..e175c05f6c --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.util.Span; + +/** + * The default {@link Document} implementation: an unmodifiable map from layer key to an + * unmodifiable annotation list. + * + *

Instances are immutable: {@link #with(LayerKey, List)} returns a new document that + * shares the unchanged layers with its ancestor, copying the map but not the layers, so + * documents grown from a common ancestor share their layer lists. The text is captured + * as a {@link String} at construction, so a mutable {@link CharSequence} handed to + * {@link #empty(CharSequence)} cannot change the document afterwards. That immutability + * makes instances safe to share between threads.

+ */ +final class ImmutableDocument implements Document { + + private final String text; + private final Map, List>> layers; + + private ImmutableDocument(String text, Map, List>> layers) { + this.text = text; + this.layers = Collections.unmodifiableMap(layers); + } + + /** + * Creates a document without any layers. + * + * @param text The original document text, captured as its content at this moment. + * Must not be {@code null}. + * @return An empty {@link ImmutableDocument}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static ImmutableDocument empty(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + return new ImmutableDocument(text.toString(), Collections.emptyMap()); + } + + /** {@inheritDoc} */ + @Override + public CharSequence text() { + return text; + } + + /** {@inheritDoc} */ + @Override + @SuppressWarnings("unchecked") + public List> get(LayerKey layer) { + if (layer == null) { + throw new IllegalArgumentException("layer must not be null"); + } + final List> annotations = layers.get(layer); + if (annotations == null) { + return List.of(); + } + // This cast is safe because with(LayerKey, List) verified every value against the + // key's type when the layer was inserted. + return (List>) (List) annotations; + } + + /** {@inheritDoc} */ + @Override + public Set> layers() { + // The unmodifiable map exposes an unmodifiable key set and caches it, so this + // accessor allocates no wrapper per call. + return layers.keySet(); + } + + /** {@inheritDoc} */ + @Override + public Document with(LayerKey layer, List> annotations) { + if (layer == null) { + throw new IllegalArgumentException("layer must not be null"); + } + if (annotations == null) { + throw new IllegalArgumentException("annotations must not be null"); + } + if (layers.containsKey(layer)) { + throw new IllegalArgumentException("layer is already present: " + layer); + } + validate(layer, annotations); + final Map, List>> grown = new LinkedHashMap<>(layers); + grown.put(layer, List.copyOf(annotations)); + return new ImmutableDocument(text, grown); + } + + /** + * {@inheritDoc} + * This implementation copies the layer map once, not once per added layer. + */ + @Override + public Document merge(Document other, DuplicateLayerPolicy duplicateLayers) { + if (other == null) { + throw new IllegalArgumentException("other must not be null"); + } + if (duplicateLayers == null) { + throw new IllegalArgumentException("duplicateLayers must not be null"); + } + if (!text.contentEquals(other.text())) { + throw new IllegalArgumentException( + "merge requires both documents to carry the same text"); + } + final Map, List>> combined = new LinkedHashMap<>(layers); + for (final LayerKey layer : other.layers()) { + if (combined.containsKey(layer)) { + if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL + && get(layer).equals(other.get(layer))) { + continue; + } + throw new IllegalArgumentException(duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL + ? "layer is present on both documents with differing contents: " + layer + : "layer is already present: " + layer); + } + combined.put(layer, copyValidated(layer, other)); + } + if (combined.size() == layers.size()) { + return this; + } + return new ImmutableDocument(text, combined); + } + + /** + * {@return a validated immutable copy of one of {@code from}'s layers, capturing the + * key's value type} + */ + private List> copyValidated(LayerKey layer, Document from) { + final List> annotations = from.get(layer); + if (annotations == null) { + throw new IllegalArgumentException("annotations must not be null"); + } + validate(layer, annotations); + return List.copyOf(annotations); + } + + /** + * Checks one layer's annotations against the key's contract: no null elements, values + * assignable to the key's type, spans present and within the text bounds under a + * positional key, absent under a document-scoped key. + * + * @throws IllegalArgumentException Thrown if any check fails; the message names the + * layer. + */ + private void validate(LayerKey layer, List> annotations) { + for (final Annotation annotation : annotations) { + if (annotation == null) { + throw new IllegalArgumentException("annotations must not contain null: " + layer); + } + if (!layer.type().isInstance(annotation.value())) { + throw new IllegalArgumentException("value of type " + + annotation.value().getClass().getName() + " does not match layer " + layer); + } + final Span span = annotation.span(); + if (layer.scope() == LayerKey.Scope.POSITIONAL) { + if (span == null) { + throw new IllegalArgumentException( + "positional layer " + layer + " requires a span on every annotation"); + } + if (span.getEnd() > text.length()) { + throw new IllegalArgumentException("span " + span + " exceeds the text length " + + text.length() + " in layer " + layer); + } + } else if (span != null) { + throw new IllegalArgumentException( + "document-scoped layer " + layer + " must not carry spans"); + } + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java new file mode 100644 index 0000000000..e29870a06c --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Objects; + +import opennlp.tools.util.StringUtil; + +/** + * Identifies one annotation layer of a {@link Document} and carries the type of that + * layer's annotation values, so reading a layer back is statically typed. + * + *

The key space is deliberately open: any producer may define new keys in its own + * package, and the container never enumerates them. Two keys are equal when their id, + * their value type, and their {@link Scope} are equal, so independently created + * constants for the same layer interoperate. Standard keys for the toolkit's own + * results live in {@link Layers}.

+ * + *

A key declares its {@link Scope}: a {@link Scope#POSITIONAL positional} key + * guarantees a span on every annotation, and a {@link Scope#DOCUMENT document-scoped} + * key carries whole-document values without spans, for example a language id or a + * category distribution. The scope is declared per key, never per annotation, so + * consumers of a positional layer never null-check a span.

+ * + * @param The type of the annotation values stored under this key. + * + * @since 3.0.0 + */ +public final class LayerKey { + + /** How the annotations of a layer relate to the document text. */ + public enum Scope { + /** Every annotation of the layer is anchored to a span of the text. */ + POSITIONAL, + /** The layer's values describe the document as a whole and carry no spans. */ + DOCUMENT + } + + private final String id; + private final Class type; + private final Scope scope; + + private LayerKey(String id, Class type, Scope scope) { + this.id = id; + this.type = type; + this.scope = scope; + } + + /** + * Creates a {@link LayerKey}. + * + * @param id The layer identifier, for example {@code opennlp:tokens}. Keys defined by + * the toolkit carry the {@code opennlp:} prefix, an extension uses its own + * prefix, and a bare id is legal for an application-local layer. Must not + * be {@code null} or blank. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A {@link Scope#POSITIONAL positional} {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + public static LayerKey of(String id, Class type) { + return key(id, type, Scope.POSITIONAL); + } + + /** + * Creates a {@link Scope#DOCUMENT document-scoped} {@link LayerKey} for values that + * describe the document as a whole, for example a language id, a category + * distribution, or provenance. Annotations under such a key carry no span. + * + * @param id The layer identifier, following the same prefix rules as + * {@link #of(String, Class)}. Must not be {@code null} or blank. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A document-scoped {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + public static LayerKey document(String id, Class type) { + return key(id, type, Scope.DOCUMENT); + } + + /** + * Validates the components and creates the key. + * + * @param id The layer identifier. + * @param type The value class. + * @param scope The declared scope. + * @param The type of the annotation values. + * @return The {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + private static LayerKey key(String id, Class type, Scope scope) { + if (id == null || StringUtil.isBlank(id)) { + throw new IllegalArgumentException("id must not be null or blank"); + } + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + return new LayerKey<>(id, type, scope); + } + + /** + * @return The layer identifier. Never {@code null}. + */ + public String id() { + return id; + } + + /** + * @return The class of the annotation values stored under this key. Never {@code null}. + */ + public Class type() { + return type; + } + + /** + * @return The declared scope of the layer. Never {@code null}. + */ + public Scope scope() { + return scope; + } + + /** {@inheritDoc} */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LayerKey other)) { + return false; + } + return id.equals(other.id) && type.equals(other.type) && scope == other.scope; + } + + /** {@inheritDoc} */ + @Override + public int hashCode() { + return Objects.hash(id, type, scope); + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return id + '<' + type.getSimpleName() + '>'; + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java new file mode 100644 index 0000000000..0257afc384 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import opennlp.tools.util.StringUtil; + +/** + * The standard {@link LayerKey layer keys} for the results the toolkit produces itself. + * + *

This class is a convenience, not a registry: the key space stays open, and any + * producer may define further keys in its own package. New capabilities must never + * require an addition here to function.

+ * + *

Namespace rule: every key the toolkit itself defines carries the + * {@code opennlp:} id prefix. An extension defines its keys under its own prefix, and + * a bare id without a prefix is legal for an application-local layer, so ids from + * independent producers cannot collide. Toolkit keys are created through + * {@link #key(String, Class)} and {@link #documentKey(String, Class)}, which apply the + * prefix, so no producer spells it.

+ * + *

Gold versus predicted: a corpus may carry a hand-annotated version of a layer + * beside a produced one. The convention is a {@code gold:} id prefix on the same key + * scheme, for example {@code gold:opennlp:tokens} beside {@code opennlp:tokens}. + * Because adding a layer is once-only, competing versions of a layer always live under + * distinct keys and never replace each other.

+ * + *

Placement rule: this class holds only the keys of the core linguistic layers + * every pipeline shares (sentences, tokens, tags, entities). A capability-specific + * layer's key lives on the annotator that provides it, for example the lemma layer's + * key on its adapter, so adding a capability never touches this class.

+ * + * @since 3.0.0 + */ +public final class Layers { + + /** The id prefix of every key the toolkit defines. */ + private static final String NAMESPACE = "opennlp:"; + + /** + * Sentence boundaries; each annotation covers one sentence and carries its text. + */ + public static final LayerKey SENTENCES = key("sentences", String.class); + + /** + * Token boundaries; each annotation covers one token and carries its text. + */ + public static final LayerKey TOKENS = key("tokens", String.class); + + /** + * Part-of-speech tags; one annotation per token, aligned with {@link #TOKENS} by + * position, carrying the tag. + */ + public static final LayerKey POS_TAGS = key("pos", String.class); + + /** + * Named entities; each annotation covers one mention and carries the entity type as + * its value. The annotation's span carries offsets only. + */ + public static final LayerKey ENTITIES = key("entities", String.class); + + /** + * Creates a {@link LayerKey.Scope#POSITIONAL positional} key in the toolkit's + * {@code opennlp:} namespace, for a layer the toolkit itself produces. + * + * @param name The layer name without a namespace, for example {@code tokens}. Must + * not be {@code null}, blank, or contain {@code ':'}. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A key whose id is the name under the {@code opennlp:} prefix. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or + * contains {@code ':'}, or {@code type} is {@code null}. + */ + public static LayerKey key(String name, Class type) { + return LayerKey.of(NAMESPACE + validName(name), type); + } + + /** + * Creates a {@link LayerKey.Scope#DOCUMENT document-scoped} key in the toolkit's + * {@code opennlp:} namespace, for a whole-document value the toolkit itself produces. + * + * @param name The layer name without a namespace, for example {@code language}. Must + * not be {@code null}, blank, or contain {@code ':'}. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A document-scoped key whose id is the name under the {@code opennlp:} + * prefix. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or + * contains {@code ':'}, or {@code type} is {@code null}. + */ + public static LayerKey documentKey(String name, Class type) { + return LayerKey.document(NAMESPACE + validName(name), type); + } + + /** + * Validates a namespace-free layer name. + * + * @param name The name to validate. + * @return The validated name. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or + * contains {@code ':'}. + */ + private static String validName(String name) { + if (name == null || StringUtil.isBlank(name)) { + throw new IllegalArgumentException("name must not be null or blank"); + } + if (name.indexOf(':') >= 0) { + throw new IllegalArgumentException( + "name must not contain ':', the namespace is applied by this factory: " + name); + } + return name; + } + + private Layers() { + // Not instantiated; this class provides constants and static key factories only. + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 98cca59891..82041f7764 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -268,6 +268,30 @@ public static boolean isEmpty(CharSequence theString) { return theString.length() == 0; } + /** + * Determines whether a {@link CharSequence} is blank: empty, or made up entirely of + * code points that {@link #isWhitespace(int)} accepts. Unlike + * {@link String#isBlank()}, this follows the toolkit's whitespace definition, which + * includes the no-break spaces the JDK predicate leaves out, so a value spelled + * entirely from them cannot pass a blank check as content. Unlike + * {@link #isUnicodeBlank(CharSequence)}, it resolves through the active + * {@link WhitespaceMode} and does not treat {@code null} as blank. + * + * @param theString The {@link CharSequence} to examine. Must not be {@code null}. + * @return {@code true} if {@code theString} is empty or all whitespace. + * @throws NullPointerException Thrown if {@code theString} is {@code null}. + */ + public static boolean isBlank(CharSequence theString) { + for (int i = 0; i < theString.length(); ) { + final int codePoint = Character.codePointAt(theString, i); + if (!isWhitespace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + /** * Get the minimum of three values. * diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java new file mode 100644 index 0000000000..3229e339f8 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants and accessors of {@link DependencyGraph} and {@link DependencyArc}. + */ +public class DependencyGraphTest { + + /** The three-token graph shared by the accessor tests. */ + private static DependencyGraph sample() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencyGraph graph = sample(); + assertEquals(3, graph.size()); + assertEquals(1, graph.headOf(0)); + assertEquals(2, graph.headOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, graph.headOf(2)); + assertEquals("nsubj", graph.relationOf(1)); + assertEquals(2, graph.root()); + } + + @Test + void testArcsAreInTokenOrder() { + final List arcs = sample().arcs(); + assertEquals(3, arcs.size()); + assertEquals(new DependencyArc(1, 0, "det"), arcs.get(0)); + assertEquals(new DependencyArc(2, 1, "nsubj"), arcs.get(1)); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 2, "root"), arcs.get(2)); + } + + @Test + void testEqualsAndHashCode() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + assertNotEquals(sample(), DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"amod", "nsubj", "root"})); + } + + @Test + void testInputArraysAreCopied() { + final int[] heads = {1, -1}; + final String[] relations = {"nsubj", "root"}; + final DependencyGraph graph = DependencyGraph.of(heads, relations); + heads[0] = 0; + relations[0] = "det"; + assertEquals(1, graph.headOf(0)); + assertEquals("nsubj", graph.relationOf(0)); + } + + @Test + void testNullArraysThrow() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(null, new String[] {"root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, null)); + } + + @Test + void testEmptyGraphThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[0], new String[0])); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, new String[] {"root", "nsubj"})); + } + + @Test + void testRootCountIsEnforced() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, 0}, new String[] {"a", "b"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1, -1}, new String[] {"root", "root"})); + } + + @Test + void testSelfHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {0, -1}, new String[] {"a", "root"})); + } + + @Test + void testOutOfRangeHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {2, -1}, new String[] {"a", "root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-3, -1}, new String[] {"a", "root"})); + } + + @Test + void testBlankRelationThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {" ", "root"})); + // blankness follows the toolkit whitespace definition, which covers the no-break + // space U+00A0 that the JDK predicate leaves out + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {"\u00A0", "root"})); + // and a label that only looks unusual is still content + assertEquals("nmod:poss", DependencyGraph.of(new int[] {1, -1}, + new String[] {"nmod:poss", "root"}).relationOf(0)); + } + + @Test + void testIndexBoundsThrow() { + final DependencyGraph graph = sample(); + assertThrows(IllegalArgumentException.class, () -> graph.headOf(-1)); + assertThrows(IllegalArgumentException.class, () -> graph.relationOf(3)); + } + + @Test + void testArcValidation() { + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(0, 0, "root")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, -1, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(-2, 0, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, " ")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, "\u00A0")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, null)); + assertEquals("det", new DependencyArc(1, 0, "det").relation()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java new file mode 100644 index 0000000000..6be16f4817 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants of {@link DependencySample}. + */ +public class DependencySampleTest { + + private static final String[] TOKENS = {"the", "dog", "barks"}; + private static final String[] TAGS = {"DT", "NN", "VBZ"}; + + /** The graph matching {@link #TOKENS} and {@link #TAGS}. */ + private static DependencyGraph graph() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencySample sample = new DependencySample(TOKENS, TAGS, graph()); + assertArrayEquals(TOKENS, sample.getTokens()); + assertArrayEquals(TAGS, sample.getTags()); + assertEquals(graph(), sample.getGraph()); + } + + @Test + void testEquals() { + assertEquals(new DependencySample(TOKENS, TAGS, graph()), + new DependencySample(TOKENS, TAGS, graph())); + } + + @Test + void testNullArgumentsThrow() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(null, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, null, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, TAGS, null)); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[] {"one"}, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, new String[] {"DT"}, graph())); + } + + @Test + void testEmptySampleThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[0], new String[0], graph())); + } + + @Test + void testInputArraysAreCopied() { + final String[] tokens = TOKENS.clone(); + final DependencySample sample = new DependencySample(tokens, TAGS, graph()); + tokens[0] = "a"; + assertEquals("the", sample.getTokens()[0]); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java new file mode 100644 index 0000000000..bef5db118f --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link DocumentAnnotators} support methods directly: the required-layer + * check's exact rejection messages and the per-sentence walk's slicing, skipping, and + * loud rejection of a token outside every sentence. The adapter tests exercise the same + * behavior through the annotators; this class pins the helpers as public API on their + * own. + */ +public class DocumentAnnotatorsTest { + + @Test + void testRequireLayersAcceptsPresentLayers() { + final Document document = Document.of("the") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()); + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); + DocumentAnnotators.requireLayers(document); + } + + @Test + void testRequireLayersRejectsNullDocument() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnnotators.requireLayers(null, Layers.TOKENS)); + assertEquals("document must not be null", e.getMessage()); + } + + /** + * Verifies that an absent layer is rejected with the shared message naming the first + * absent layer in the order the caller listed them. + */ + @Test + void testRequireLayersNamesTheFirstAbsentLayer() { + final Document document = Document.of("the").with(Layers.TOKENS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnnotators.requireLayers(document, + Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS)); + assertEquals("document lacks the required layer opennlp:sentences", + e.getMessage()); + } + + /** + * Verifies the walk contract: each sentence receives the contiguous run of tokens its + * span encloses with the run's first token layer position, and a sentence without + * tokens is skipped rather than reported as an empty run. + */ + @Test + void testForEachSentenceSlicesContiguousRuns() { + final List> sentences = List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 11), "!"), + new Annotation<>(new Span(12, 21), "Bob sits.")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(12, 15), "Bob"), + new Annotation<>(new Span(16, 21), "sits.")); + + final List firsts = new ArrayList<>(); + final List> runs = new ArrayList<>(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + firsts.add(first); + runs.add(List.of(words)); + }); + + assertEquals(List.of(0, 2), firsts); + assertEquals(List.of( + List.of("Ana", "runs."), + List.of("Bob", "sits.")), runs); + } + + @Test + void testForEachSentenceRejectsTokenOutsideEverySentence() { + final List> sentences = List.of( + new Annotation<>(new Span(0, 9), "Ana runs.")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob")); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + })); + assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + } + + @Test + void testForEachSentenceOverEmptyLayersConsumesNothing() { + final List> runs = new ArrayList<>(); + DocumentAnnotators.forEachSentence(List.of(), List.of(), + (first, words) -> runs.add(List.of(words))); + assertTrue(runs.isEmpty()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java new file mode 100644 index 0000000000..27a6a1948f --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -0,0 +1,644 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins down the observable contract of the document container at its edges: key + * equality, layer ordering, span boundary cases, list immutability, the exact rejection + * messages, and the analyzer's build-time validation. Every expected value in this class + * is asserted exactly so any behavioral drift is caught, not just gross breakage. + */ +public class DocumentContractTest { + + private static final LayerKey WORDS = LayerKey.of("words", String.class); + + /** + * Verifies that two independently created keys with the same id and the same type are + * equal, hash alike, and therefore address the same layer, while remaining distinct + * instances. This is what lets separately compiled producers agree on a layer without + * sharing a constant. + */ + @Test + void testKeysWithSameIdAndTypeAddressTheSameLayer() { + final LayerKey first = LayerKey.of("words", String.class); + final LayerKey second = LayerKey.of("words", String.class); + assertNotSame(first, second); + assertEquals(first, second); + assertEquals(first.hashCode(), second.hashCode()); + assertEquals("words", first.toString()); + + final Document document = Document.of("the") + .with(first, List.of(new Annotation<>(new Span(0, 3), "the"))); + // Reading through the other, equal key yields the very layer added above. + assertEquals(1, document.get(second).size()); + assertEquals("the", document.get(second).get(0).value()); + // A duplicate add through the equal key is rejected like any duplicate. + assertThrows(IllegalArgumentException.class, () -> document.with(second, List.of())); + } + + /** + * Verifies that two keys sharing an id but differing in value type are unequal and + * denote two independent layers that can coexist on one document. + */ + @Test + void testKeysWithSameIdButDifferentTypesAreDifferentLayers() { + final LayerKey asString = LayerKey.of("marks", String.class); + final LayerKey asInteger = LayerKey.of("marks", Integer.class); + assertNotEquals(asString, asInteger); + + final Document document = Document.of("ab") + .with(asString, List.of(new Annotation<>(new Span(0, 1), "a"))) + .with(asInteger, List.of(new Annotation<>(new Span(1, 2), 7))); + assertEquals(Set.of(asString, asInteger), document.layers()); + assertEquals("a", document.get(asString).get(0).value()); + assertEquals(7, document.get(asInteger).get(0).value()); + } + + /** + * Verifies the per-key scope contract: a document-scoped layer carries span-less + * values and round-trips them, a spanned annotation under a document-scoped key is + * rejected with a message naming the layer, a span-less annotation under a + * positional key is rejected likewise, and two keys differing only in scope are + * unequal and never address the same layer. + */ + @Test + void testDocumentScopedLayersCarrySpanlessValues() { + final LayerKey language = LayerKey.document("language", String.class); + assertEquals(LayerKey.Scope.DOCUMENT, language.scope()); + assertEquals(LayerKey.Scope.POSITIONAL, WORDS.scope()); + + final Document document = Document.of("the dog") + .with(language, List.of(Annotation.of("eng"))); + assertEquals(1, document.get(language).size()); + assertEquals("eng", document.get(language).get(0).value()); + assertNull(document.get(language).get(0).span()); + + final IllegalArgumentException spanned = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(language, + List.of(new Annotation<>(new Span(0, 3), "eng")))); + assertEquals("document-scoped layer language must not carry spans", + spanned.getMessage()); + + final IllegalArgumentException spanless = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(WORDS, List.of(Annotation.of("the")))); + assertEquals("positional layer words requires a span on every annotation", + spanless.getMessage()); + + final LayerKey positionalTwin = LayerKey.of("language", String.class); + assertNotEquals(language, positionalTwin); + final Document both = Document.of("the") + .with(language, List.of(Annotation.of("eng"))) + .with(positionalTwin, List.of(new Annotation<>(new Span(0, 3), "the"))); + assertEquals(2, both.layers().size()); + } + + /** + * Verifies the toolkit-namespace factories: {@link Layers#key(String, Class)} and + * {@link Layers#documentKey(String, Class)} apply the {@code opennlp:} prefix and + * yield keys equal to independently spelled ones, and a name that is null, blank, or + * already carries a namespace is rejected. + */ + @Test + void testToolkitKeysCarryTheNamespacePrefix() { + assertEquals(LayerKey.of("opennlp:things", String.class), + Layers.key("things", String.class)); + assertEquals("opennlp:things", Layers.key("things", String.class).id()); + assertEquals(Layers.TOKENS, Layers.key("tokens", String.class)); + + final LayerKey whole = Layers.documentKey("language", String.class); + assertEquals("opennlp:language", whole.id()); + assertEquals(LayerKey.Scope.DOCUMENT, whole.scope()); + + assertThrows(IllegalArgumentException.class, () -> Layers.key(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> Layers.key(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> Layers.key("things", null)); + final IllegalArgumentException nested = assertThrows(IllegalArgumentException.class, + () -> Layers.key("opennlp:things", String.class)); + assertEquals("name must not contain ':', the namespace is applied by this factory: " + + "opennlp:things", nested.getMessage()); + assertThrows(IllegalArgumentException.class, + () -> Layers.documentKey("app:x", String.class)); + } + + /** + * Verifies the gold-layer convention: a hand-annotated version of a layer lives + * under the {@code gold:} prefixed key beside the produced layer, both are readable + * independently, and the once-only rule keeps either from replacing the other. + */ + @Test + void testGoldLayerLivesBesideThePredictedLayer() { + final LayerKey gold = LayerKey.of("gold:" + Layers.TOKENS.id(), String.class); + final Document document = Document.of("the dog") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "the"))) + .with(gold, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + assertEquals("gold:opennlp:tokens", gold.id()); + assertEquals(1, document.get(Layers.TOKENS).size()); + assertEquals(2, document.get(gold).size()); + assertThrows(IllegalArgumentException.class, + () -> document.with(gold, List.of())); + } + + /** + * Verifies that a layer preserves the insertion order of its annotations: the + * container does not sort by span, so a producer that wants span order must supply + * span order. + */ + @Test + void testLayerPreservesInsertionOrder() { + final Document document = Document.of("the dog") + .with(WORDS, List.of( + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(0, 3), "the"))); + final List> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals(new Span(4, 7), words.get(0).span()); + assertEquals("dog", words.get(0).value()); + assertEquals(new Span(0, 3), words.get(1).span()); + assertEquals("the", words.get(1).value()); + } + + /** + * Verifies that several annotations may share one span within a layer, for example + * alternative readings of the same region, and all of them are retained in order. + */ + @Test + void testAnnotationsWithIdenticalSpansAreAllRetained() { + final Document document = Document.of("bank") + .with(WORDS, List.of( + new Annotation<>(new Span(0, 4), "institution"), + new Annotation<>(new Span(0, 4), "riverside"))); + final List> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals("institution", words.get(0).value()); + assertEquals("riverside", words.get(1).value()); + assertEquals(words.get(0).span(), words.get(1).span()); + } + + /** + * Verifies that zero-length spans are accepted anywhere within the bounds, including + * at the very end of the text where start and end equal the text length. + */ + @Test + void testZeroLengthSpansAreAccepted() { + final Document document = Document.of("ab") + .with(WORDS, List.of( + new Annotation<>(new Span(1, 1), "between"), + new Annotation<>(new Span(2, 2), "at the end"))); + final List> words = document.get(WORDS); + assertEquals(new Span(1, 1), words.get(0).span()); + assertEquals(new Span(2, 2), words.get(1).span()); + assertEquals(0, words.get(0).span().length()); + } + + /** + * Verifies that spans are indexed in {@code char} units, like {@link Span} itself: a + * supplementary-plane character counts as two, so a span over it covers the whole + * surrogate pair and the char-based text length is what bounds a span. + */ + @Test + void testSpansAreCharIndexedOverSupplementaryCharacters() { + // U+1F600, a supplementary-plane character, is two chars in the text + final String text = "\uD83D\uDE00 ok"; + final Document document = Document.of(text) + .with(WORDS, List.of( + new Annotation<>(new Span(0, 2), "emoji"), + new Annotation<>(new Span(3, 5), "ok"))); + final List> words = document.get(WORDS); + assertEquals("\uD83D\uDE00", + words.get(0).span().getCoveredText(document.text()).toString()); + assertEquals("ok", words.get(1).span().getCoveredText(document.text()).toString()); + assertThrows(IllegalArgumentException.class, () -> Document.of("\uD83D\uDE00") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "past the end")))); + } + + /** + * Verifies that a span reaching past the end of the text is rejected on insertion + * with a message naming the span, the text length, and the layer. + */ + @Test + void testSpanBeyondTextLengthIsRejectedWithExactMessage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(WORDS, + List.of(new Annotation<>(new Span(0, 4), "the?")))); + assertEquals("span [0..4) exceeds the text length 3 in layer words", + e.getMessage()); + } + + /** + * Verifies that adding a layer under a key that is already present is rejected with a + * message naming the offending layer. + */ + @Test + void testDuplicateLayerIsRejectedWithExactMessage() { + final Document document = Document.of("the").with(WORDS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> document.with(WORDS, List.of())); + assertEquals("layer is already present: words", e.getMessage()); + } + + /** + * Verifies that reading an absent layer yields an empty, unmodifiable list rather + * than {@code null}, so callers can iterate without a presence check. + */ + @Test + void testAbsentLayerReadsAsUnmodifiableEmptyList() { + final List> absent = Document.of("the").get(WORDS); + assertTrue(absent.isEmpty()); + assertThrows(UnsupportedOperationException.class, + () -> absent.add(new Annotation<>(new Span(0, 3), "the"))); + } + + /** + * Verifies that the list returned for a present layer is unmodifiable and detached + * from the caller's input list: mutating the input after the add does not change the + * document. + */ + @Test + void testPresentLayerListIsUnmodifiableAndDetachedFromInput() { + final List> input = new ArrayList<>(); + input.add(new Annotation<>(new Span(0, 3), "the")); + final Document document = Document.of("the").with(WORDS, input); + + final List> words = document.get(WORDS); + assertThrows(UnsupportedOperationException.class, () -> words.remove(0)); + assertThrows(UnsupportedOperationException.class, + () -> words.add(new Annotation<>(new Span(0, 3), "the"))); + + input.clear(); + assertEquals(1, document.get(WORDS).size()); + } + + /** + * Verifies that the layer key set exposed by a document cannot be mutated by callers. + */ + @Test + void testLayerKeySetIsUnmodifiable() { + final Document document = Document.of("the").with(WORDS, List.of()); + assertThrows(UnsupportedOperationException.class, () -> document.layers().clear()); + } + + /** + * Verifies that an analyzer whose annotator requires a layer no earlier annotator + * provides fails at build time with a message naming the annotator and the missing + * layer. The annotator under test overrides {@code toString()} so the whole message + * can be asserted exactly. + */ + @Test + void testUnsatisfiedRequirementFailsAtBuildTimeWithExactMessage() { + final DocumentAnnotator needsTags = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + throw new IllegalStateException("must never run; the pipeline must not build"); + } + + @Override + public Set> requires() { + return Set.of(Layers.POS_TAGS); + } + + @Override + public Set> provides() { + return Set.of(WORDS); + } + + @Override + public String toString() { + return "tag-consumer"; + } + }; + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder().add(needsTags); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, builder::build); + assertEquals("annotator tag-consumer requires layer opennlp:pos," + + " which no earlier annotator provides", e.getMessage()); + } + + /** + * Verifies that an analyzer whose annotators would provide the same layer twice fails + * at build time with a message naming the layer and the positions of both providers, + * instead of crashing midway through the first document. + */ + @Test + void testDuplicateProviderFailsAtBuildTimeWithExactMessage() { + final DocumentAnnotator provider = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + throw new IllegalStateException("must never run; the pipeline must not build"); + } + + @Override + public Set> provides() { + return Set.of(WORDS); + } + }; + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(provider).add(provider); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, builder::build); + assertEquals("annotators at positions 0 and 1 both provide layer words", + e.getMessage()); + } + + /** + * Verifies that building an analyzer without any annotator fails with a message + * stating that a pipeline needs at least one annotator. + */ + @Test + void testEmptyPipelineFailsWithExactMessage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnalyzer.builder().build()); + assertEquals("a pipeline needs at least one annotator", e.getMessage()); + } + + /** + * Verifies that a caller can add a new layer type without changing the document + * container. + */ + @Test + void testCustomLayerNeedsNoContainerChange() { + record Sentiment(String polarity, double score) { + } + final LayerKey sentiment = LayerKey.of("sentiment", Sentiment.class); + final DocumentAnnotator annotator = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + final Span all = new Span(0, document.text().length()); + return document.with(sentiment, + List.of(new Annotation<>(all, new Sentiment("positive", 0.9d)))); + } + + @Override + public Set> provides() { + return Set.of(sentiment); + } + }; + final Document document = DocumentAnalyzer.builder().add(annotator).build() + .analyze("good dog"); + assertEquals("positive", document.get(sentiment).get(0).value().polarity()); + } + + /** + * Verifies that the value type travels through {@link LayerKey}: a layer added under + * an {@code Integer} key reads back as {@code Annotation}, so its values + * participate in arithmetic without a cast, and a mismatched value can never enter + * the layer in the first place. + */ + @Test + void testValueTypeTravelsThroughTheKey() { + final LayerKey counts = LayerKey.of("counts", Integer.class); + final Document document = Document.of("ab cd") + .with(counts, List.of( + new Annotation<>(new Span(0, 2), 2), + new Annotation<>(new Span(3, 5), 40))); + int sum = 0; + for (final Annotation count : document.get(counts)) { + sum += count.value(); + } + assertEquals(42, sum); + + // The insertion-time check backs the typed read: a raw-typed caller cannot place a + // String under the Integer key. + @SuppressWarnings({"unchecked", "rawtypes"}) + final LayerKey raw = (LayerKey) LayerKey.of("counts2", Integer.class); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> Document.of("ab").with(raw, + List.of(new Annotation<>(new Span(0, 2), "not a number")))); + assertEquals("value of type java.lang.String does not match layer counts2", + e.getMessage()); + } + + /** + * Verifies that merge joins two documents grown independently over the same text and + * leaves both sources untouched. Text content decides equality, not the + * {@link CharSequence} implementation. + */ + @Test + void testMergeJoinsLayersOfDocumentsOverTheSameText() { + final LayerKey lengths = LayerKey.of("lengths", Integer.class); + final Document words = Document.of("the dog") + .with(WORDS, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + final Document counted = Document.of(new StringBuilder("the dog")) + .with(lengths, List.of( + new Annotation<>(new Span(0, 3), 3), + new Annotation<>(new Span(4, 7), 3))); + + final Document merged = words.merge(counted); + + assertEquals("the dog", merged.text().toString()); + assertEquals(Set.of(WORDS, lengths), merged.layers()); + assertEquals("the", merged.get(WORDS).get(0).value()); + assertEquals(3, merged.get(lengths).get(0).value().intValue()); + assertEquals(Set.of(WORDS), words.layers()); + assertEquals(Set.of(lengths), counted.layers()); + } + + /** + * Verifies that merge rejects a null argument, a document over a different text even + * when their layers are disjoint, and a layer key present on both documents, naming + * the offending key. + */ + @Test + void testMergeRejectsNullDifferentTextAndDuplicateLayers() { + final Document words = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"))); + + final IllegalArgumentException nullOther = assertThrows(IllegalArgumentException.class, + () -> words.merge(null)); + assertEquals("other must not be null", nullOther.getMessage()); + + final IllegalArgumentException differentText = assertThrows( + IllegalArgumentException.class, () -> words.merge(Document.of("the cat"))); + assertEquals("merge requires both documents to carry the same text", + differentText.getMessage()); + + final IllegalArgumentException duplicate = assertThrows(IllegalArgumentException.class, + () -> words.merge(words)); + assertEquals("layer is already present: words", duplicate.getMessage()); + } + + /** + * Verifies that {@link Document.DuplicateLayerPolicy#KEEP_EQUAL} keeps one copy of a + * layer both documents rebuilt identically while still joining the disjoint layers. + */ + @Test + void testMergeKeepingEqualLayersToleratesIdenticalCopies() { + final List> tokens = List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog")); + final LayerKey lengths = LayerKey.of("lengths", Integer.class); + final Document words = Document.of("the dog").with(WORDS, tokens); + final Document recounted = Document.of("the dog") + .with(WORDS, tokens) + .with(lengths, List.of( + new Annotation<>(new Span(0, 3), 3), + new Annotation<>(new Span(4, 7), 3))); + + final Document merged = words.merge(recounted, Document.DuplicateLayerPolicy.KEEP_EQUAL); + + assertEquals(Set.of(WORDS, lengths), merged.layers()); + // The shared layer is kept once, not concatenated. + assertEquals(2, merged.get(WORDS).size()); + assertEquals(2, merged.get(lengths).size()); + } + + /** + * Verifies that {@link Document.DuplicateLayerPolicy#KEEP_EQUAL} still rejects a layer + * whose two copies differ, naming the key, and rejects a null policy. + */ + @Test + void testMergeKeepingEqualLayersRejectsDifferingCopiesAndNullPolicy() { + final Document words = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"))); + final Document retokenized = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 7), "the dog"))); + + final IllegalArgumentException differing = assertThrows(IllegalArgumentException.class, + () -> words.merge(retokenized, Document.DuplicateLayerPolicy.KEEP_EQUAL)); + assertEquals("layer is present on both documents with differing contents: words", + differing.getMessage()); + + final IllegalArgumentException nullPolicy = assertThrows(IllegalArgumentException.class, + () -> words.merge(Document.of("the dog"), null)); + assertEquals("duplicateLayers must not be null", nullPolicy.getMessage()); + } + + /** + * Verifies that a document implementation that does not override merge gets the same + * semantics from the interface default: disjoint layers join, a layer both documents + * rebuilt identically is kept once under KEEP_EQUAL, and differing copies are rejected + * with the same message the default implementation produces. + */ + @Test + void testMergeDefaultImplementationServesForeignDocuments() { + final List> tokens = List.of(new Annotation<>(new Span(0, 3), "the")); + final LayerKey lengths = LayerKey.of("lengths", Integer.class); + final Document words = new DelegatingDocument(Document.of("the dog").with(WORDS, tokens)); + final Document counted = Document.of("the dog") + .with(WORDS, tokens) + .with(lengths, List.of(new Annotation<>(new Span(0, 3), 3))); + + final Document merged = words.merge(counted, Document.DuplicateLayerPolicy.KEEP_EQUAL); + assertEquals(Set.of(WORDS, lengths), merged.layers()); + assertEquals(1, merged.get(WORDS).size()); + + final Document retokenized = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 7), "the dog"))); + final IllegalArgumentException differing = assertThrows(IllegalArgumentException.class, + () -> words.merge(retokenized, Document.DuplicateLayerPolicy.KEEP_EQUAL)); + assertEquals("layer is present on both documents with differing contents: words", + differing.getMessage()); + } + + /** + * Verifies that merge validates the layers it takes from the other document instead of + * trusting them: a foreign implementation can hand out annotations that were never + * checked, and an out-of-bounds span among them is rejected by name. + */ + @Test + void testMergeRevalidatesLayersOfForeignDocuments() { + final Document words = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"))); + final LayerKey stale = LayerKey.of("stale", String.class); + final Document unvalidated = new UnvalidatedDocument("the dog", stale, + List.of(new Annotation<>(new Span(0, 99), "out of bounds"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> words.merge(unvalidated)); + assertEquals("span [0..99) exceeds the text length 7 in layer stale", + e.getMessage()); + } + + /** + * A pass-through wrapper that overrides none of the interface defaults, so calling + * merge on it runs the interface's default implementation. + */ + private record DelegatingDocument(Document delegate) implements Document { + + @Override + public CharSequence text() { + return delegate.text(); + } + + @Override + public List> get(LayerKey layer) { + return delegate.get(layer); + } + + @Override + public Set> layers() { + return delegate.layers(); + } + + @Override + public Document with(LayerKey layer, List> annotations) { + return new DelegatingDocument(delegate.with(layer, annotations)); + } + } + + /** + * A document whose single layer bypassed all validation, standing in for a foreign + * implementation that does not enforce the layer contract itself. + */ + private record UnvalidatedDocument(String rawText, LayerKey key, + List> annotations) + implements Document { + + @Override + public CharSequence text() { + return rawText; + } + + @Override + @SuppressWarnings("unchecked") + public List> get(LayerKey layer) { + return key.equals(layer) ? (List>) (List) annotations : List.of(); + } + + @Override + public Set> layers() { + return Set.of(key); + } + + @Override + public Document with(LayerKey layer, List> annotations) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java new file mode 100644 index 0000000000..787c9d8edb --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link Document} container: typed layer access, copy-on-add immutability, + * and the insertion-time validation that protects the layer invariants. + */ +public class DocumentTest { + + private static final LayerKey WORDS = LayerKey.of("words", String.class); + private static final LayerKey NUMBERS = LayerKey.of("numbers", Integer.class); + + @Test + void testEmptyDocument() { + final Document document = Document.of("the dog"); + assertEquals("the dog", document.text()); + assertTrue(document.layers().isEmpty()); + assertTrue(document.get(WORDS).isEmpty()); + } + + /** + * Verifies that the text is captured at construction: mutating a + * {@link StringBuilder} handed to {@link Document#of(CharSequence)} does not reach + * the document, so the span bounds validated on insertion stay valid for its + * lifetime. + */ + @Test + void testTextIsCapturedAtConstruction() { + final StringBuilder mutable = new StringBuilder("the dog"); + final Document document = Document.of(mutable) + .with(WORDS, List.of(new Annotation<>(new Span(4, 7), "dog"))); + mutable.setLength(0); + assertEquals("the dog", document.text().toString()); + assertEquals("dog", + document.get(WORDS).get(0).span().getCoveredText(document.text()).toString()); + } + + @Test + void testWithAddsATypedLayer() { + final Document document = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + assertEquals(Set.of(WORDS), document.layers()); + final List> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals("dog", words.get(1).value()); + assertEquals(new Span(4, 7), words.get(1).span()); + } + + @Test + void testWithIsCopyOnAdd() { + final Document empty = Document.of("42"); + final Document grown = empty.with(NUMBERS, + List.of(new Annotation<>(new Span(0, 2), 42))); + assertTrue(empty.layers().isEmpty()); + assertEquals(Set.of(NUMBERS), grown.layers()); + // unchanged layers are shared, not copied + final Document both = grown.with(WORDS, List.of()); + assertSame(grown.get(NUMBERS), both.get(NUMBERS)); + } + + @Test + void testEqualKeysFromDifferentConstantsInteroperate() { + final Document document = Document.of("the") + .with(LayerKey.of("words", String.class), + List.of(new Annotation<>(new Span(0, 3), "the"))); + assertEquals(1, document.get(WORDS).size()); + assertNotEquals(WORDS, LayerKey.of("words", CharSequence.class)); + } + + @Test + void testDuplicateLayerThrows() { + final Document document = Document.of("the").with(WORDS, List.of()); + assertThrows(IllegalArgumentException.class, () -> document.with(WORDS, List.of())); + } + + @Test + void testSpanBeyondTextThrows() { + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(WORDS, List.of(new Annotation<>(new Span(0, 4), "the?")))); + } + + @Test + void testValueTypeIsCheckedOnInsertion() { + // a raw-typed caller cannot smuggle a mismatched value past the layer type + @SuppressWarnings({"unchecked", "rawtypes"}) + final LayerKey raw = (LayerKey) NUMBERS; + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(raw, List.of(new Annotation<>(new Span(0, 3), "not a number")))); + } + + @Test + void testNullArgumentsThrow() { + final Document document = Document.of("the"); + assertThrows(IllegalArgumentException.class, () -> Document.of(null)); + assertThrows(IllegalArgumentException.class, () -> document.get(null)); + assertThrows(IllegalArgumentException.class, () -> document.with(null, List.of())); + assertThrows(IllegalArgumentException.class, () -> document.with(WORDS, null)); + } + + @Test + void testAnnotationValidation() { + // a span-less annotation is legal to build; the container judges it per key scope + assertThrows(IllegalArgumentException.class, () -> new Annotation<>(new Span(0, 3), null)); + assertThrows(IllegalArgumentException.class, () -> Annotation.of(null)); + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(WORDS, List.of(Annotation.of("the")))); + } + + @Test + void testLayerKeyValidation() { + assertThrows(IllegalArgumentException.class, () -> LayerKey.of(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.of(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.of("words", null)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.document(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.document(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.document("lang", null)); + } +} diff --git a/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md new file mode 100644 index 0000000000..3052d88553 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md @@ -0,0 +1,48 @@ + + +# Universal Dependencies treebanks for the dependency parser evaluation + +The dependency parser's unit tests are fully self-contained, but its accuracy evaluation runs against a real Universal Dependencies treebank that the user downloads. Apache OpenNLP bundles no treebank data, distributes none, and ships no models trained on it; the treebanks are used to reproduce accuracy numbers on your own machine. + +## Getting a treebank + +Every UD treebank lives in its own repository under `github.com/UniversalDependencies`, with its splits named `-ud-train.conllu`, `-dev`, and `-test`. The helper next to this file clones one shallowly and lays the splits out under the names the evaluation expects: + +``` +./download-ud-treebank.sh UD_English-EWT /tmp/ud-ewt +``` + +produces `/tmp/ud-ewt/train.conllu` and `/tmp/ud-ewt/test.conllu`. Any treebank that publishes both splits works the same way. + +## Running the gated evaluation + +`ConlluDependencyParserEvalTest` is disabled unless the `opennlp.depparse.ud.dir` system property points at a directory containing `train.conllu` and `test.conllu`: + +``` +./mvnw -pl opennlp-core/opennlp-formats test \ + -Dtest=ConlluDependencyParserEvalTest \ + -Dopennlp.depparse.ud.dir=/tmp/ud-ewt +``` + +Without the property the test reports as skipped, which is why a plain build never needs network access or external data. + +Two properties of the parser worth knowing when reading the numbers: multiword-token sentences are kept because the CoNLL-U reader recovers their dependency rows, and non-projective training sentences are skipped, since the arc-standard transition system cannot derive them; the skip count is inherent to the algorithm, not data loss in the reader. + +## Licensing + +Each treebank carries its own license, stated in its repository README, and downloading one means accepting those terms yourself. The annotations of `UD_English-EWT`, for example, are licensed under CC BY-SA 4.0. The project's handling: treebanks are benchmark inputs on the user's machine only; no treebank data enters the source tree or any release artifact, and the project publishes no models trained on share-alike data. If you train and distribute your own model from a treebank, checking that treebank's terms is your responsibility. diff --git a/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh new file mode 100755 index 0000000000..f229b57a31 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fetches one Universal Dependencies treebank and lays its splits out the way the +# gated dependency-parser evaluation expects: /train.conllu and +# /test.conllu. See README-ud-treebanks.md in this directory for the +# evaluation command and the licensing notes; each treebank carries its own license, +# which you accept by downloading it. Nothing is bundled with Apache OpenNLP. + +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 + echo "" >&2 + echo " treebank-repository a repository name under github.com/UniversalDependencies," >&2 + echo " for example UD_English-EWT" >&2 + echo " target-dir where train.conllu and test.conllu are placed" >&2 + exit 2 +} + +[ $# -ne 2 ] && usage +treebank="$1" +target="$2" + +# Clone shallowly into a temporary directory; only the .conllu files are kept. +clone="$(mktemp -d)" +trap 'rm -rf "${clone}"' EXIT +echo "cloning ${treebank}" +git clone --quiet --depth 1 "https://github.com/UniversalDependencies/${treebank}.git" \ + "${clone}/${treebank}" + +mkdir -p "${target}" +for split in train test; do + # UD names its files -ud-.conllu; the code prefix varies per + # treebank, so match on the stable -ud- suffix. + found="" + for f in "${clone}/${treebank}/"*"-ud-${split}.conllu"; do + [ -e "$f" ] && found="$f" && break + done + if [ -z "${found}" ]; then + echo "no *-ud-${split}.conllu in ${treebank}; the treebank may not publish" >&2 + echo "that split (some hide test data or ship dev only)" >&2 + exit 1 + fi + cp "${found}" "${target}/${split}.conllu" + echo "wrote ${target}/${split}.conllu ($(wc -l < "${target}/${split}.conllu") lines)" +done + +echo "" +echo "run the gated evaluation with:" +echo " ./mvnw -pl opennlp-core/opennlp-formats test \\" +echo " -Dtest=ConlluDependencyParserEvalTest \\" +echo " -Dopennlp.depparse.ud.dir=${target}" diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java new file mode 100644 index 0000000000..f1df227783 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; + +/** + * Reads {@link DependencySample samples} directly from + * CoNLL-U content, mapping + * the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * + *

The file is parsed raw, deliberately not through {@link ConlluStream}: that stream + * merges multiword token ranges with their syntactic words, which suits the token and + * lemma views but destroys the dependency annotation of every sentence containing a + * contraction. Here range lines and empty nodes are dropped while their syntactic words + * are kept, so contraction-bearing sentences train and evaluate normally. Sentences + * whose annotation is incomplete or invalid, for example an underscore head, are + * skipped and counted; the count is logged once the stream is exhausted.

+ * + * @since 3.0.0 + */ +public class ConlluDependencySampleStream implements ObjectStream { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencySampleStream.class); + + private static final int COLUMNS = 10; + private static final int FORM = 1; + private static final int UPOS = 3; + private static final int XPOS = 4; + private static final int HEAD = 6; + private static final int DEPREL = 7; + + private final InputStreamFactory in; + private final int tagColumn; + + private BufferedReader reader; + private int skipped; + + /** + * Initializes the stream. + * + * @param in The CoNLL-U content. Must not be {@code null}. + * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must + * not be {@code null}. + * @throws IOException Thrown if opening the content fails. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public ConlluDependencySampleStream(InputStreamFactory in, ConlluTagset tagset) + throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + if (tagset == null) { + throw new IllegalArgumentException("tagset must not be null"); + } + this.in = in; + this.tagColumn = tagset == ConlluTagset.U ? UPOS : XPOS; + this.reader = open(); + } + + @Override + public DependencySample read() throws IOException { + List words; + while (!(words = nextSentence()).isEmpty()) { + final DependencySample sample = convert(words); + if (sample != null) { + return sample; + } + skipped++; + } + if (skipped > 0) { + logger.warn("Skipped {} sentence(s) without a complete basic dependency annotation.", + skipped); + skipped = 0; + } + return null; + } + + /** + * Reads the syntactic word lines of the next sentence: comments, multiword token + * ranges, and empty nodes are dropped; an empty list means the end of the content. + * + *

Sentences are separated by any line {@link StringUtil#isBlank(CharSequence)} + * accepts, so a separator carrying a stray no-break space still separates rather than + * reaching the word line parser.

+ * + * @return The word lines of the next sentence, or an empty list at the end of the + * content. Never {@code null}. + * @throws IOException Thrown if reading fails or a word line has too few columns. + */ + private List nextSentence() throws IOException { + final List words = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + if (StringUtil.isBlank(line)) { + if (!words.isEmpty()) { + return words; + } + continue; + } + if (line.charAt(0) == '#') { + continue; + } + final String[] fields = splitFields(line); + if (fields.length < COLUMNS) { + throw new IOException("not a CoNLL-U word line: " + line); + } + final String id = fields[0]; + if (id.indexOf('-') < 0 && id.indexOf('.') < 0) { + words.add(fields); + } + } + return words; + } + + /** + * Splits a CoNLL-U word line into its tab-delimited fields, retaining empty fields. + * + * @param line The line to split. + * @return The fields in source order. Never {@code null}. + */ + private String[] splitFields(String line) { + final List fields = new ArrayList<>(); + int fieldStart = 0; + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) == '\t') { + fields.add(line.substring(fieldStart, i)); + fieldStart = i + 1; + } + } + fields.add(line.substring(fieldStart)); + return fields.toArray(String[]::new); + } + + /** + * Converts one sentence into a sample. + * + * @param words The word lines of the sentence. + * @return The converted sample, or {@code null} when the sentence's annotation is + * unusable, for example an underscore head or a graph that is not a tree. + */ + private DependencySample convert(List words) { + final int n = words.size(); + final String[] tokens = new String[n]; + final String[] tags = new String[n]; + final int[] heads = new int[n]; + final String[] relations = new String[n]; + for (int i = 0; i < n; i++) { + final String[] word = words.get(i); + tokens[i] = word[FORM]; + tags[i] = word[tagColumn]; + relations[i] = word[DEPREL]; + try { + heads[i] = Integer.parseInt(word[HEAD]) - 1; + } catch (NumberFormatException e) { + return null; + } + } + try { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } catch (IllegalArgumentException e) { + return null; + } + } + + @Override + public void reset() throws IOException, UnsupportedOperationException { + reader.close(); + reader = open(); + skipped = 0; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + /** + * Opens a fresh UTF-8 reader over the content. + * + * @return A reader positioned at the start of the content. Never {@code null}. + * @throws IOException Thrown if opening the content fails. + */ + private BufferedReader open() throws IOException { + return new BufferedReader( + new InputStreamReader(in.createInputStream(), StandardCharsets.UTF_8)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java new file mode 100644 index 0000000000..fd0934ee9f --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.MarkableFileInputStreamFactory; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Trains the greedy arc-standard parser on a Universal Dependencies treebank and scores + * it on the treebank's test split, reporting UAS and LAS. + * + *

Runs only when {@code opennlp.depparse.ud.dir} names a directory containing + * {@code train.conllu} and {@code test.conllu} (a UD treebank's splits, renamed or + * linked). The data is downloaded by the runner and never enters the repository; check + * the treebank's own license before training models for distribution. The assertion is + * a low regression floor; the logged scores are the measurement.

+ */ +public class ConlluDependencyParserEvalTest { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencyParserEvalTest.class); + + @Test + @EnabledIfSystemProperty(named = "opennlp.depparse.ud.dir", matches = ".+") + void testTrainAndScoreOnUniversalDependencies() throws IOException { + final Path dir = Path.of(System.getProperty("opennlp.depparse.ud.dir")); + + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 5); + final long trainStart = System.currentTimeMillis(); + final DependencyModel model; + try (ConlluDependencySampleStream train = samples(dir.resolve("train.conllu"))) { + model = DependencyParserME.train("eng", train, parameters); + } + logger.info("trained in {} ms", System.currentTimeMillis() - trainStart); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new DependencyParserME(model)); + try (ConlluDependencySampleStream test = samples(dir.resolve("test.conllu"))) { + evaluator.evaluate(test); + } + logger.info("UAS {} LAS {} over {} tokens", + evaluator.getUas(), evaluator.getLas(), evaluator.getWordCount()); + + // a regression floor, far below any plausible result; the log line is the measurement + assertTrue(evaluator.getUas() > 0.6d, "UAS regressed below the floor"); + assertTrue(evaluator.getLas() > 0.5d, "LAS regressed below the floor"); + } + + /** Opens a sample stream over one CoNLL-U split using the universal tagset. */ + private static ConlluDependencySampleStream samples(Path conllu) throws IOException { + final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); + return new ConlluDependencySampleStream(in, ConlluTagset.U); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java new file mode 100644 index 0000000000..960653a778 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Demonstrates the full dependency parsing workflow on a self-contained fixture: read + * gold sentences from CoNLL-U content, train a {@link DependencyParserME}, parse a + * sentence, inspect the resulting {@link DependencyGraph}, and persist the model. + * + *

The fixture holds three tiny sentences inline, so the test needs no external data. + * A real treebank provides thousands of sentences; here each fixture sentence is + * repeated to give the trainer the same evidence many times, which lets the model + * memorize the fixture and makes every expected value exact.

+ */ +public class ConlluDependencyParserUsageTest { + + /** + * Joins the ten CoNLL-U columns of one word line with tabs. + * + * @param fields The column values; exactly ten are expected by the format. + * @return The joined word line. Never {@code null}. + */ + private static String line(String... fields) { + return String.join("\t", fields); + } + + /** + * The training fixture: three gold sentences in CoNLL-U form. The {@code HEAD} column + * is one-based with {@code 0} marking the root; the reader converts it to the + * zero-based convention of {@link DependencyGraph}. + */ + private static final String CONLLU = String.join("\n", + "# text = the dog barks", + line("1", "the", "the", "DET", "DT", "_", "2", "det", "_", "_"), + line("2", "dog", "dog", "NOUN", "NN", "_", "3", "nsubj", "_", "_"), + line("3", "barks", "bark", "VERB", "VBZ", "_", "0", "root", "_", "_"), + "", + "# text = dogs bark", + line("1", "dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "", + "# text = she eats fish", + line("1", "she", "she", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "eats", "eat", "VERB", "VBZ", "_", "0", "root", "_", "_"), + line("3", "fish", "fish", "NOUN", "NN", "_", "2", "obj", "_", "_"), + "") + "\n"; + + private static DependencyModel model; + private static DependencyParserME parser; + + /** + * Reads the fixture sentences through the CoNLL-U reader. + * + * @return One sample per fixture sentence, in file order. Never {@code null}. + * @throws IOException Thrown if reading the in-memory content fails. + */ + private static List readFixture() throws IOException { + final InputStreamFactory in = + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + final List samples = new ArrayList<>(); + try (ConlluDependencySampleStream stream = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + DependencySample sample; + while ((sample = stream.read()) != null) { + samples.add(sample); + } + } + return samples; + } + + /** + * Trains the parser once for all tests: read the fixture, repeat it for evidence, + * and hand the samples to the trainer. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final List fixture = readFixture(); + final List trainingSamples = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + trainingSamples.addAll(fixture); + } + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(trainingSamples), parameters); + parser = new DependencyParserME(model); + } + + @Test + void testReaderDeliversTheGoldAnnotation() throws IOException { + final List fixture = readFixture(); + assertEquals(3, fixture.size()); + final DependencySample first = fixture.get(0); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), first.getGraph()); + assertEquals("NOUN", first.getTags()[1]); + } + + @Test + void testParseAssignsHeadsAndRelations() { + // Parsing takes the tokens and their part-of-speech tags; the result names, for + // every token, its head token and the relation between the two. + final DependencyGraph parse = parser.parse( + new String[] {"the", "dog", "barks"}, new String[] {"DET", "NOUN", "VERB"}); + assertEquals(1, parse.headOf(0)); + assertEquals("det", parse.relationOf(0)); + assertEquals(2, parse.headOf(1)); + assertEquals("nsubj", parse.relationOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, parse.headOf(2)); + assertEquals("root", parse.relationOf(2)); + assertEquals(2, parse.root()); + } + + @Test + void testEvaluatorScoresTheParserAgainstGoldSamples() throws IOException { + // The evaluator parses each gold sentence and accumulates the two standard scores; + // on its own training fixture the memorizing model is exact on all eight tokens. + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(readFixture())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(8, evaluator.getWordCount()); + } + + @Test + void testPersistedModelParsesLikeTheOriginal(@TempDir Path dir) throws IOException { + // A trained model is saved to a file and loaded back like any other tool model; the + // reloaded parser must produce the exact same parse as the original. + final Path file = dir.resolve("en-depparse.bin"); + model.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRON", "VERB", "NOUN"})); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java new file mode 100644 index 0000000000..460e531a7f --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests that the raw reader maps the basic dependency columns, keeps the syntactic + * words of multiword tokens while dropping the range line itself, and skips sentences + * without a usable annotation. + */ +public class ConlluDependencySampleStreamTest { + + /** Joins the ten CoNLL-U columns of one word line with tabs. */ + private static String line(String... fields) { + return String.join("\t", fields); + } + + private static final String CONLLU = String.join("\n", + "# sent_id = test-1", + "# text = He bought the bonds", + line("1", "He", "he", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "bought", "buy", "VERB", "VBD", "_", "0", "root", "_", "_"), + line("3", "the", "the", "DET", "DT", "_", "4", "det", "_", "_"), + line("4", "bonds", "bond", "NOUN", "NNS", "_", "2", "obj", "_", "_"), + "", + "# sent_id = test-2", + "# text = Broken", + line("1", "Broken", "broken", "ADJ", "JJ", "_", "_", "_", "_", "_"), + "", + "# sent_id = test-3", + "# text = im Haus", + line("1-2", "im", "_", "_", "_", "_", "_", "_", "_", "_"), + line("1", "in", "in", "ADP", "APPR", "_", "2", "case", "_", "_"), + line("2", "Haus", "Haus", "NOUN", "NN", "_", "0", "root", "_", "_"), + "", + "# sent_id = test-4", + "# text = Dogs bark", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "") + "\n"; + + /** An in-memory factory over the shared fixture. */ + private static InputStreamFactory factory() { + return () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + } + + /** A stream over the shared fixture using the universal tagset. */ + private static ConlluDependencySampleStream stream() throws IOException { + return new ConlluDependencySampleStream(factory(), ConlluTagset.U); + } + + @Test + void testReadsSamplesKeepsContractionsAndSkipsUnusableSentences() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + assertArrayEquals(new String[] {"PRON", "VERB", "DET", "NOUN"}, first.getTags()); + assertEquals(1, first.getGraph().headOf(0)); + assertEquals(DependencyArc.ROOT_HEAD, first.getGraph().headOf(1)); + assertEquals(3, first.getGraph().headOf(2)); + assertEquals("obj", first.getGraph().relationOf(3)); + + // the underscore-head sentence is skipped; the contraction sentence is KEPT, + // with the range line dropped and its syntactic words intact + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"in", "Haus"}, second.getTokens()); + assertEquals(1, second.getGraph().headOf(0)); + assertEquals("case", second.getGraph().relationOf(0)); + + final DependencySample third = samples.read(); + assertNotNull(third); + assertArrayEquals(new String[] {"Dogs", "bark"}, third.getTokens()); + + assertNull(samples.read()); + } + } + + @Test + void testResetRestartsTheStream() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + assertNotNull(samples.read()); + samples.reset(); + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + } + } + + @Test + void testXposTagsetSelectsTheOtherColumn() throws IOException { + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(factory(), ConlluTagset.X)) { + assertArrayEquals(new String[] {"PRP", "VBD", "DT", "NNS"}, + samples.read().getTags()); + } + } + + @Test + void testMalformedLineFailsLoud() { + final InputStreamFactory bad = () -> new ByteArrayInputStream( + "1\ttoo\tfew\tcolumns\n".getBytes(StandardCharsets.UTF_8)); + assertThrows(IOException.class, + () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); + } + + @Test + void testSemanticallyInvalidAnnotationIsSkippedNotFatal() throws IOException { + // Structurally well-formed lines whose annotation cannot form a valid tree, here an + // out-of-range head and a rootless cycle, skip the sentence instead of failing, so + // one broken sentence cannot abort reading a large treebank. + final String content = String.join("\n", + line("1", "far", "far", "ADV", "RB", "_", "5", "advmod", "_", "_"), + line("2", "off", "off", "ADP", "RP", "_", "0", "root", "_", "_"), + "", + line("1", "loop", "loop", "NOUN", "NN", "_", "2", "dep", "_", "_"), + line("2", "back", "back", "ADV", "RB", "_", "1", "dep", "_", "_"), + "", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample onlyValid = samples.read(); + assertNotNull(onlyValid); + assertArrayEquals(new String[] {"Fine"}, onlyValid.getTokens()); + assertEquals(DependencyArc.ROOT_HEAD, onlyValid.getGraph().headOf(0)); + assertNull(samples.read()); + } + } + + @Test + void testSeparatorLineOfNonBreakingSpaceSeparatesSentences() throws IOException { + // A separator line carrying a stray no-break space is still a separator: OpenNLP + // counts U+00A0 as whitespace, so such a line must not reach the word-line parser + // and abort the stream. + final String content = String.join("\n", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "\u00A0", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"Dogs", "bark"}, first.getTokens()); + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"Fine"}, second.getTokens()); + assertNull(samples.read()); + } + } + + @Test + void testEmptyContentYieldsNoSample() throws IOException { + final InputStreamFactory in = () -> new ByteArrayInputStream(new byte[0]); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + assertNull(samples.read()); + } + } + + @Test + void testValidation() { + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(null, ConlluTagset.U)); + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(factory(), null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java new file mode 100644 index 0000000000..8bd4a17a44 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.chunker; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link Chunker} to the document pipeline: reads {@link Layers#SENTENCES}, + * {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and provides {@link #CHUNKS}, one + * annotation per phrase chunk carrying the chunk type, for example {@code NP} or + * {@code VP}, on the span from its first to its last token. + * + *

Each sentence is chunked separately with its tokens and tags as one sequence, the + * way the chunker contract expects its input. A chunker's spans index tokens within the + * sentence; the adapter maps them onto the token spans, which already refer to the + * original text, so a chunk covers exactly the text of its tokens. Chunks are emitted in + * text order.

+ * + *

The adapter holds no per-call state; it is as thread-safe as the chunker it + * wraps.

+ * + * @since 3.0.0 + */ +public final class ChunkerAnnotator implements DocumentAnnotator { + + /** + * Phrase chunks; each annotation covers one chunk and carries its type, ordered by + * text position. + */ + public static final LayerKey CHUNKS = Layers.key("chunks", String.class); + + private final Chunker chunker; + + /** + * Initializes the adapter. + * + * @param chunker The chunker to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code chunker} is {@code null}. + */ + public ChunkerAnnotator(Chunker chunker) { + if (chunker == null) { + throw new IllegalArgumentException("chunker must not be null"); + } + this.chunker = chunker; + } + + /** + * Chunks the document sentence by sentence and adds the {@link #CHUNKS} layer. + * + *

The required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty chunk layer. The token and tag + * layers must be aligned one to one.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES}, {@link Layers#TOKENS}, and + * {@link Layers#POS_TAGS} layers, with every token lying inside a + * sentence. + * @return A new {@link Document} with the {@link #CHUNKS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, a + * required layer is absent, the token and tag layers differ in size, a token + * lies outside every sentence, or the chunker returns a span outside the + * sentence, an empty span, or a span without a type. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS, + Layers.POS_TAGS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final List> chunks = new ArrayList<>(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final String[] sentenceTags = new String[words.length]; + for (int i = 0; i < words.length; i++) { + sentenceTags[i] = tags.get(first + i).value(); + } + for (final Span chunk : chunker.chunkAsSpans(words, sentenceTags)) { + if (chunk.getStart() < 0 || chunk.getEnd() > words.length + || chunk.getStart() >= chunk.getEnd()) { + throw new IllegalArgumentException("chunker returned chunk " + chunk + + " outside the sentence's " + words.length + " tokens"); + } + if (chunk.getType() == null) { + throw new IllegalArgumentException( + "chunker returned chunk " + chunk + " without a type"); + } + chunks.add(new Annotation<>(new Span( + tokens.get(first + chunk.getStart()).span().getStart(), + tokens.get(first + chunk.getEnd() - 1).span().getEnd()), chunk.getType())); + } + }); + return document.with(CHUNKS, chunks); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(CHUNKS); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java new file mode 100644 index 0000000000..596ef4d2d4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + + +/** + * The static oracle for the arc-standard system: derives the transition sequence that + * reproduces a gold {@link DependencyGraph}. + * + *

An arc is only created once all dependents of the token being attached have been + * collected, which is the arc-standard correctness condition. The oracle is defined for + * projective trees only; a non-projective gold graph has no arc-standard derivation and + * is rejected.

+ * + * @since 3.0.0 + */ +public final class ArcStandardOracle { + + private ArcStandardOracle() { + // This class only exposes static derivation methods and is never instantiated. + } + + /** + * Derives the gold transition sequence for a graph. + * + * @param gold The gold dependency graph. Must not be {@code null} and must be + * projective. + * @return The transitions that rebuild {@code gold} from the start configuration, in + * order. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code gold} is {@code null} or not + * projective. + */ + public static List transitions(DependencyGraph gold) { + if (gold == null) { + throw new IllegalArgumentException("gold must not be null"); + } + final int n = gold.size(); + final int[] goldDependents = new int[n]; + for (int i = 0; i < n; i++) { + final int head = gold.headOf(i); + if (head >= 0) { + goldDependents[head]++; + } + } + + final ArcStandardState state = new ArcStandardState(n); + final List transitions = new ArrayList<>(2 * n); + while (!state.isTerminal()) { + final Transition next = nextTransition(gold, goldDependents, state); + if (next == null) { + throw new IllegalArgumentException( + "gold graph has no arc-standard derivation (non-projective): " + gold); + } + state.apply(next); + transitions.add(next); + } + return transitions; + } + + /** + * Picks the gold transition for the current configuration, or {@code null} when the + * configuration is stuck, which only happens for non-projective input. + */ + private static Transition nextTransition(DependencyGraph gold, int[] goldDependents, + ArcStandardState state) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + if (s1 >= 0 && gold.headOf(s1) == s0) { + final Transition leftArc = Transition.leftArc(gold.relationOf(s1)); + if (state.canApply(leftArc)) { + return leftArc; + } + } + if (s0 >= 0 && s1 != ArcStandardState.NONE && gold.headOf(s0) == s1 + && state.assignedDependents(s0) == goldDependents[s0]) { + final Transition rightArc = Transition.rightArc(gold.relationOf(s0)); + if (state.canApply(rightArc)) { + return rightArc; + } + } + if (state.canApply(Transition.SHIFT)) { + return Transition.SHIFT; + } + return null; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java new file mode 100644 index 0000000000..afa2646e9a --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.Arrays; + +/** + * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining + * tokens, and the arcs assigned so far. The arc-standard transition system is described + * in Nivre (2004). + * + *

The stack bottom holds the artificial root, exposed as {@link #ROOT}. Positions that + * do not exist, such as the second stack element in the initial configuration, are exposed + * as {@link #NONE}. A right arc from the artificial root is only applicable once the buffer + * is empty and the root is the only other stack element, which guarantees every completed + * parse has exactly one sentence root.

+ * + *

Instances are confined to a single parse and must not be shared between threads.

+ * + * @since 3.0.0 + */ +public final class ArcStandardState { + + /** The stack value representing the artificial root node. */ + public static final int ROOT = -1; + + /** The value returned for stack or buffer positions that do not exist. */ + public static final int NONE = -2; + + private final int tokenCount; + private final int[] stack; + private final int[] heads; + private final String[] relations; + private final int[] assignedDependents; + private final int[] leftmostDependents; + private final int[] rightmostDependents; + + private int top; + private int bufferFront; + + /** + * Initializes the start configuration for a sentence: the artificial root on the stack + * and every token in the buffer. + * + * @param tokenCount The number of tokens in the sentence. Must be greater than zero. + * @throws IllegalArgumentException Thrown if {@code tokenCount} is not positive. + */ + public ArcStandardState(int tokenCount) { + if (tokenCount <= 0) { + throw new IllegalArgumentException("tokenCount must be positive: " + tokenCount); + } + this.tokenCount = tokenCount; + this.stack = new int[tokenCount + 1]; + this.stack[0] = ROOT; + this.top = 0; + this.bufferFront = 0; + this.heads = new int[tokenCount]; + this.relations = new String[tokenCount]; + this.assignedDependents = new int[tokenCount]; + this.leftmostDependents = new int[tokenCount]; + this.rightmostDependents = new int[tokenCount]; + Arrays.fill(this.leftmostDependents, NONE); + Arrays.fill(this.rightmostDependents, NONE); + } + + /** + * Deep-copies {@code source}; used only by {@link #copy()}. + */ + private ArcStandardState(ArcStandardState source) { + this.tokenCount = source.tokenCount; + this.stack = source.stack.clone(); + this.heads = source.heads.clone(); + this.relations = source.relations.clone(); + this.assignedDependents = source.assignedDependents.clone(); + this.leftmostDependents = source.leftmostDependents.clone(); + this.rightmostDependents = source.rightmostDependents.clone(); + this.top = source.top; + this.bufferFront = source.bufferFront; + } + + /** + * Creates an independent copy of this configuration, so alternatives can be advanced + * side by side during search. + * + * @return A copy that can be advanced without affecting this state. Never {@code null}. + */ + public ArcStandardState copy() { + return new ArcStandardState(this); + } + + /** + * @return {@code true} if the buffer is empty and only the artificial root remains on + * the stack, so the parse is complete. + */ + public boolean isTerminal() { + return bufferFront == tokenCount && top == 0; + } + + /** + * Checks whether a transition may be applied in the current configuration. + * + * @param transition The transition to check. Must not be {@code null}. + * @return {@code true} if {@link #apply(Transition)} would succeed. + * @throws IllegalArgumentException Thrown if {@code transition} is {@code null}. + */ + public boolean canApply(Transition transition) { + if (transition == null) { + throw new IllegalArgumentException("transition must not be null"); + } + return switch (transition.type()) { + case SHIFT -> bufferFront < tokenCount; + case LEFT_ARC -> top >= 2; + case RIGHT_ARC -> top >= 2 || (top == 1 && bufferFront == tokenCount); + }; + } + + /** + * Applies a transition, updating stack, buffer, and arcs. + * + * @param transition The transition to apply. Must not be {@code null} and must be + * applicable per {@link #canApply(Transition)}. + * @throws IllegalArgumentException Thrown if the transition is {@code null} or not + * applicable in the current configuration. + */ + public void apply(Transition transition) { + if (!canApply(transition)) { + throw new IllegalArgumentException("transition not applicable: " + transition + + " in " + this); + } + switch (transition.type()) { + case SHIFT -> { + top++; + stack[top] = bufferFront++; + } + case LEFT_ARC -> { + final int dependent = stack[top - 1]; + attach(stack[top], dependent, transition.label()); + stack[top - 1] = stack[top]; + top--; + } + case RIGHT_ARC -> { + attach(stack[top - 1], stack[top], transition.label()); + top--; + } + default -> throw new IllegalArgumentException("unsupported type: " + transition.type()); + } + } + + /** + * Records the arc from {@code head} to {@code dependent} and updates the dependent + * bookkeeping of {@code head} when it is a token rather than the artificial root. + * + * @param head The head token index, or {@link #ROOT} for the artificial root. + * @param dependent The zero-based index of the token being attached. + * @param relation The relation label of the arc. + */ + private void attach(int head, int dependent, String relation) { + heads[dependent] = head; + relations[dependent] = relation; + if (head >= 0) { + assignedDependents[head]++; + if (leftmostDependents[head] == NONE || dependent < leftmostDependents[head]) { + leftmostDependents[head] = dependent; + } + if (rightmostDependents[head] == NONE || dependent > rightmostDependents[head]) { + rightmostDependents[head] = dependent; + } + } + } + + /** + * Retrieves a stack element counted from the top. + * + * @param fromTop Zero for the top element, one for the element below it, and so on. + * Must not be negative. + * @return The token index at that position, {@link #ROOT} for the artificial root, or + * {@link #NONE} if the position does not exist. + * @throws IllegalArgumentException Thrown if {@code fromTop} is negative. + */ + public int stack(int fromTop) { + if (fromTop < 0) { + throw new IllegalArgumentException("fromTop must not be negative: " + fromTop); + } + final int position = top - fromTop; + return position < 0 ? NONE : stack[position]; + } + + /** + * Retrieves a buffer element counted from the front. + * + * @param fromFront Zero for the next token to be shifted, one for the token after it, + * and so on. Must not be negative. + * @return The token index at that position, or {@link #NONE} if the position does not + * exist. + * @throws IllegalArgumentException Thrown if {@code fromFront} is negative. + */ + public int buffer(int fromFront) { + if (fromFront < 0) { + throw new IllegalArgumentException("fromFront must not be negative: " + fromFront); + } + final int position = bufferFront + fromFront; + return position >= tokenCount ? NONE : position; + } + + /** + * @return The number of stack elements including the artificial root. + */ + public int stackSize() { + return top + 1; + } + + /** + * @return The number of tokens still in the buffer. + */ + public int bufferSize() { + return tokenCount - bufferFront; + } + + /** + * Retrieves how many dependents have been attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The number of arcs assigned with the token as head. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int assignedDependents(int index) { + checkTokenIndex(index); + return assignedDependents[index]; + } + + /** + * Rejects a token index outside {@code [0, tokenCount)}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + private void checkTokenIndex(int index) { + if (index < 0 || index >= tokenCount) { + throw new IllegalArgumentException("token index out of range: " + index); + } + } + + /** + * Retrieves the leftmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int leftmostDependent(int index) { + checkTokenIndex(index); + return leftmostDependents[index]; + } + + /** + * Retrieves the rightmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int rightmostDependent(int index) { + checkTokenIndex(index); + return rightmostDependents[index]; + } + + /** + * Retrieves the relation a token was attached under, when it has been attached. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The relation label, or {@code null} when the token is still unattached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String assignedRelation(int index) { + checkTokenIndex(index); + return relations[index]; + } + + /** + * Builds the {@link DependencyGraph} of a completed parse. + * + * @return The parsed graph. Never {@code null}. + * @throws IllegalStateException Thrown if the parse is not yet terminal. + */ + public DependencyGraph toGraph() { + if (!isTerminal()) { + throw new IllegalStateException("parse is not terminal: " + this); + } + return DependencyGraph.of(heads, relations); + } + + @Override + public String toString() { + return "stackSize=" + stackSize() + ", bufferSize=" + bufferSize() + + ", tokenCount=" + tokenCount; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java new file mode 100644 index 0000000000..e7a7d555fa --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link DependencyParser} to the document pipeline: reads + * {@link Layers#SENTENCES}, {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and + * provides {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's + * span. + * + *

An arc's {@link DependencyArc#head()} and {@link DependencyArc#dependent()} are + * indices into the token layer, following the container's rule that annotations + * reference each other by layer and index, never by object identity.

+ * + *

Each sentence is parsed separately, the way the parser contract expects its input, + * so every sentence gets its own tree and its own root arc. The sentence-local indices + * the parser returns are shifted by the sentence's first token position, which keeps + * every arc's head and dependent a position in the document-wide token layer. Token + * spans already refer to the original document text, so anchoring an arc on its + * dependent token's span puts the arc in document coordinates.

+ * + * @since 3.0.0 + */ +public class DependencyAnnotator implements DocumentAnnotator { + + /** + * Dependency arcs; one annotation per token, aligned with {@link Layers#TOKENS} by + * position, anchored on the dependent token's span. + */ + public static final LayerKey DEPENDENCIES = + Layers.key("dependencies", DependencyArc.class); + + /** The message prefix of every absent-required-layer rejection in this adapter. */ + private static final String MISSING_LAYER = "document lacks the required layer "; + + private final DependencyParser parser; + + /** + * Initializes the adapter. + * + * @param parser The dependency parser to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyAnnotator(DependencyParser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + /** + * Parses the document sentence by sentence and adds the {@link #DEPENDENCIES} layer. + * + *

For every sentence, the tokens whose spans lie inside the sentence span are + * passed to the parser with their tags as one sequence, and the resulting + * sentence-local arcs are shifted by the sentence's first token position. Arcs are + * emitted in token order, so the new layer is aligned with {@link Layers#TOKENS} by + * position, and each arc annotation reuses the span of its dependent token. The + * required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty dependency layer, and a sentence + * containing no tokens contributes no arcs.

+ * + *

The sentence and token layers must both be in text order: the walk assigns each + * sentence the contiguous run of tokens its span encloses, so a token that appears + * before its sentence in the layer, or a token overlapping a sentence boundary, is + * reported as lying outside every sentence rather than being silently attached to a + * neighboring sentence.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, in + * text order, and a {@link Layers#POS_TAGS} layer with exactly one + * tag per token, with every token lying inside a sentence. + * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * sentence layer, the token layer, or the tag layer is absent, the tag layer + * does not have exactly one tag per token, a token lies outside every + * sentence under the text-order walk, or the parser returns a graph whose + * size differs from its sentence's token count. + */ + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + if (!document.layers().contains(Layers.SENTENCES)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.SENTENCES); + } + if (!document.layers().contains(Layers.TOKENS)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.TOKENS); + } + if (!document.layers().contains(Layers.POS_TAGS)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.POS_TAGS); + } + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final List> arcs = new ArrayList<>(tokens.size()); + // Walk the token layer once: both layers are in text order, so each sentence + // consumes the contiguous run of tokens whose spans it encloses. + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + final String[] posTags = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + posTags[i] = tags.get(first + i).value(); + } + final DependencyGraph graph = parser.parse(words, posTags); + if (graph.size() != count) { + throw new IllegalArgumentException("parser returned a graph over " + graph.size() + + " tokens for a sentence of " + count); + } + // The parser indexes within the sentence; shifting by the sentence's first token + // position turns every head and dependent into a document-wide token index. + for (final DependencyArc arc : graph.arcs()) { + final int head = arc.head() == DependencyArc.ROOT_HEAD + ? DependencyArc.ROOT_HEAD : arc.head() + first; + arcs.add(new Annotation<>(tokens.get(first + arc.dependent()).span(), + new DependencyArc(head, arc.dependent() + first, arc.relation()))); + } + } + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); + } + return document.with(DEPENDENCIES, arcs); + } + + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); + } + + @Override + public Set> provides() { + return Set.of(DEPENDENCIES); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java new file mode 100644 index 0000000000..bcd7ab732d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * Generates the classification features for one arc-standard configuration: words and + * tags of the topmost stack and frontmost buffer positions, their pairings, the partial + * structure built so far (tags and relations of the leftmost and rightmost dependents, + * valency counts), and a bucketed distance between stack top and buffer front. + * + *

Instances hold no state and are safe to share between threads.

+ * + * @since 3.0.0 + */ +public class DependencyContextGenerator { + + private static final String ROOT_VALUE = "*ROOT*"; + private static final String NONE_VALUE = "*NULL*"; + + /** Separates a word from the tag of the same position within one feature. */ + private static final char WORD_TAG_SEPARATOR = '/'; + + /** Separates the parts of a feature combining several positions. */ + private static final char POSITION_SEPARATOR = '|'; + + /** The number of features {@link #getContext(ArcStandardState, String[], String[])} emits. */ + private static final int FEATURE_COUNT = 37; + + /** Valency counts at or above this bound share one feature value. */ + private static final int MAX_VALENCY = 3; + + /** Distances at or above this bound share the {@link #LONG_DISTANCE} feature value. */ + private static final int MAX_DISTANCE = 4; + + /** The feature value standing for every distance of {@link #MAX_DISTANCE} or more. */ + private static final String LONG_DISTANCE = "4+"; + + /** + * Generates the features of the current configuration. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The feature strings. Never {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public String[] getContext(ArcStandardState state, String[] tokens, String[] tags) { + if (state == null || tokens == null || tags == null) { + throw new IllegalArgumentException("state, tokens and tags must not be null"); + } + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int s2 = state.stack(2); + final int b0 = state.buffer(0); + final int b1 = state.buffer(1); + final int b2 = state.buffer(2); + + final String s0w = word(tokens, s0); + final String s0t = tag(tags, s0); + final String s1w = word(tokens, s1); + final String s1t = tag(tags, s1); + final String s2t = tag(tags, s2); + final String b0w = word(tokens, b0); + final String b0t = tag(tags, b0); + final String b1w = word(tokens, b1); + final String b1t = tag(tags, b1); + final String b2t = tag(tags, b2); + + final String s0lct = dependentTag(state, tags, s0, true); + final String s0rct = dependentTag(state, tags, s0, false); + final String s1lct = dependentTag(state, tags, s1, true); + final String s1rct = dependentTag(state, tags, s1, false); + final String s0lcl = dependentRelation(state, s0, true); + final String s0rcl = dependentRelation(state, s0, false); + final String s1rcl = dependentRelation(state, s1, false); + + final List features = new ArrayList<>(FEATURE_COUNT); + features.add("s0w=" + s0w); + features.add("s0t=" + s0t); + features.add("s1w=" + s1w); + features.add("s1t=" + s1t); + features.add("s2t=" + s2t); + features.add("b0w=" + b0w); + features.add("b0t=" + b0t); + features.add("b1w=" + b1w); + features.add("b1t=" + b1t); + features.add("b2t=" + b2t); + features.add("s0wt=" + s0w + WORD_TAG_SEPARATOR + s0t); + features.add("s1wt=" + s1w + WORD_TAG_SEPARATOR + s1t); + features.add("b0wt=" + b0w + WORD_TAG_SEPARATOR + b0t); + features.add("s0w,b0w=" + s0w + POSITION_SEPARATOR + b0w); + features.add("s0t,b0t=" + s0t + POSITION_SEPARATOR + b0t); + features.add("s0w,b0t=" + s0w + POSITION_SEPARATOR + b0t); + features.add("s0t,b0w=" + s0t + POSITION_SEPARATOR + b0w); + features.add("s0wt,b0t=" + s0w + WORD_TAG_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s1t,s0t=" + s1t + POSITION_SEPARATOR + s0t); + features.add("s1t,s0w=" + s1t + POSITION_SEPARATOR + s0w); + features.add("s1w,s0t=" + s1w + POSITION_SEPARATOR + s0t); + features.add("s1t,s0t,b0t=" + s1t + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s0t,b0t,b1t=" + s0t + POSITION_SEPARATOR + b0t + POSITION_SEPARATOR + b1t); + features.add("s2t,s1t,s0t=" + s2t + POSITION_SEPARATOR + s1t + POSITION_SEPARATOR + s0t); + features.add("s0lct=" + s0lct); + features.add("s0rct=" + s0rct); + features.add("s1lct=" + s1lct); + features.add("s1rct=" + s1rct); + features.add("s0lcl=" + s0lcl); + features.add("s0rcl=" + s0rcl); + features.add("s1rcl=" + s1rcl); + features.add("s1t,s1rct,s0t=" + s1t + POSITION_SEPARATOR + s1rct + POSITION_SEPARATOR + s0t); + features.add("s0t,s0lct,b0t=" + s0t + POSITION_SEPARATOR + s0lct + POSITION_SEPARATOR + b0t); + features.add("s0deps=" + dependents(state, s0)); + features.add("s1deps=" + dependents(state, s1)); + final String distance = distance(s0, b0); + features.add("dist=" + distance); + features.add("dist,s0t,b0t=" + distance + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + return features.toArray(new String[0]); + } + + /** The word at a position, or the marker value for the root and absent positions. */ + private String word(String[] tokens, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; + } + + /** The tag at a position, or the marker value for the root and absent positions. */ + private String tag(String[] tags, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tags[index]; + } + + /** The tag of a token's leftmost or rightmost dependent attached so far. */ + private String dependentTag(ArcStandardState state, String[] tags, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + return tag(tags, dependent); + } + + /** The relation of a token's leftmost or rightmost dependent attached so far. */ + private String dependentRelation(ArcStandardState state, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + if (dependent < 0) { + return NONE_VALUE; + } + final String relation = state.assignedRelation(dependent); + return relation == null ? NONE_VALUE : relation; + } + + /** A token's dependent count so far, capped at {@link #MAX_VALENCY}. */ + private String dependents(ArcStandardState state, int index) { + return index < 0 ? NONE_VALUE + : Integer.toString(Math.min(state.assignedDependents(index), MAX_VALENCY)); + } + + /** The bucketed distance between stack top and buffer front, capped at {@link #MAX_DISTANCE}. */ + private String distance(int s0, int b0) { + if (s0 < 0 || b0 < 0) { + return NONE_VALUE; + } + final int distance = b0 - s0; + return distance >= MAX_DISTANCE ? LONG_DISTANCE : Integer.toString(distance); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java new file mode 100644 index 0000000000..c71c538c5f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.eval.Evaluator; +import opennlp.tools.util.eval.Mean; + +/** + * Measures the quality of a {@link DependencyParser} against gold + * {@link DependencySample samples} with the two standard scores: the unlabeled attachment + * score (UAS, the fraction of tokens with the correct head) and the labeled attachment + * score (LAS, the fraction of tokens with the correct head and relation label). + * + * @since 3.0.0 + */ +public class DependencyEvaluator extends Evaluator { + + private final DependencyParser parser; + private final Mean uas = new Mean(); + private final Mean las = new Mean(); + + /** + * Initializes a {@link DependencyEvaluator}. + * + * @param parser The parser to evaluate. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyEvaluator(DependencyParser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + /** + * {@inheritDoc} + * + *

The returned sample carries the predicted graph over the reference tokens, and + * every token of the reference contributes to both scores.

+ */ + @Override + protected DependencySample processSample(DependencySample reference) { + final DependencyGraph gold = reference.getGraph(); + final DependencyGraph predicted = parser.parse(reference.getTokens(), reference.getTags()); + for (int i = 0; i < gold.size(); i++) { + final boolean headMatches = gold.headOf(i) == predicted.headOf(i); + uas.add(headMatches ? 1 : 0); + las.add(headMatches && gold.relationOf(i).equals(predicted.relationOf(i)) ? 1 : 0); + } + return new DependencySample(reference.getTokens(), reference.getTags(), predicted); + } + + /** + * @return The unlabeled attachment score over all evaluated tokens. + */ + public double getUas() { + return uas.mean(); + } + + /** + * @return The labeled attachment score over all evaluated tokens. + */ + public double getLas() { + return las.mean(); + } + + /** + * @return The number of tokens scored so far. + */ + public long getWordCount() { + return uas.count(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java new file mode 100644 index 0000000000..87fdf27300 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.ml.model.Event; +import opennlp.tools.util.ObjectStream; + +/** + * Turns {@link DependencySample samples} into training {@link Event events}: for each + * sample the {@link ArcStandardOracle} derives the gold transitions, and every transition + * becomes one event pairing the configuration features with the encoded transition. + * + *

Samples whose graph has no arc-standard derivation, that is non-projective trees, + * are skipped and counted; the count is logged once the stream is exhausted.

+ */ +class DependencyEventStream implements ObjectStream { + + private static final Logger logger = LoggerFactory.getLogger(DependencyEventStream.class); + + private final ObjectStream samples; + private final DependencyContextGenerator contextGenerator; + private final Deque pending = new ArrayDeque<>(); + + private int skipped; + + /** + * Initializes the stream. + * + * @param samples The samples to convert. Must not be {@code null}. + * @param contextGenerator The feature generator. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + DependencyEventStream(ObjectStream samples, + DependencyContextGenerator contextGenerator) { + if (samples == null || contextGenerator == null) { + throw new IllegalArgumentException("samples and contextGenerator must not be null"); + } + this.samples = samples; + this.contextGenerator = contextGenerator; + } + + @Override + public Event read() throws IOException { + while (pending.isEmpty()) { + final DependencySample sample = samples.read(); + if (sample == null) { + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + skipped = 0; + } + return null; + } + final List transitions; + try { + transitions = ArcStandardOracle.transitions(sample.getGraph()); + } catch (IllegalArgumentException e) { + skipped++; + continue; + } + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : transitions) { + pending.add(new Event(transition.encode(), + contextGenerator.getContext(state, tokens, tags))); + state.apply(transition); + } + } + return pending.poll(); + } + + @Override + public void reset() throws IOException, UnsupportedOperationException { + samples.reset(); + pending.clear(); + skipped = 0; + } + + @Override + public void close() throws IOException { + samples.close(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java new file mode 100644 index 0000000000..8f2c608eeb --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; + +import opennlp.tools.ml.model.AbstractModel; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.model.BaseModel; + +/** + * The persisted form of a trained {@link DependencyParserME}: the transition + * classification model plus the standard model manifest, serialized and loaded through + * the same {@link BaseModel} machinery as every other tool model. + * + * @see DependencyParserME + * @since 3.0.0 + */ +public class DependencyModel extends BaseModel { + + private static final long serialVersionUID = -2928968185269611443L; + + private static final String COMPONENT_NAME = "DependencyParserME"; + static final String PARSER_MODEL_ENTRY_NAME = "depparse.model"; + + /** + * Initializes a {@link DependencyModel} from a trained transition model. + * + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. + * @param parserModel The transition classification model. Must not be {@code null}. + * @param manifestInfoEntries Additional entries for the manifest, or {@code null}. + * @throws IllegalArgumentException Thrown if {@code parserModel} is {@code null}. + */ + public DependencyModel(String languageCode, MaxentModel parserModel, + Map manifestInfoEntries) { + super(COMPONENT_NAME, languageCode, manifestInfoEntries); + if (parserModel == null) { + throw new IllegalArgumentException("parserModel must not be null"); + } + artifactMap.put(PARSER_MODEL_ENTRY_NAME, parserModel); + checkArtifactMap(); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model. + * + * @param in The stream to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(InputStream in) throws IOException { + super(COMPONENT_NAME, in); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelFile The file to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(File modelFile) throws IOException { + super(COMPONENT_NAME, modelFile); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelPath The path to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(Path modelPath) throws IOException { + super(COMPONENT_NAME, modelPath); + } + + @Override + protected void validateArtifactMap() throws InvalidFormatException { + super.validateArtifactMap(); + if (!(artifactMap.get(PARSER_MODEL_ENTRY_NAME) instanceof AbstractModel)) { + throw new InvalidFormatException("The " + PARSER_MODEL_ENTRY_NAME + + " artifact is missing or not a supported transition model."); + } + } + + /** + * @return The transition classification model. Never {@code null}. + */ + public MaxentModel getParserModel() { + return (MaxentModel) artifactMap.get(PARSER_MODEL_ENTRY_NAME); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java new file mode 100644 index 0000000000..c565e61724 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import opennlp.tools.ml.EventTrainer; +import opennlp.tools.ml.TrainerFactory; +import opennlp.tools.ml.TrainerFactory.TrainerType; +import opennlp.tools.ml.model.Event; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.TrainingParameters; + +/** + * A greedy transition-based {@link DependencyParser}: a maximum entropy classifier picks + * the next arc-standard {@link Transition} for each configuration until the parse is + * complete, always taking the highest scoring transition that is applicable. + * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see DependencyParser + * @since 3.0.0 + */ +public class DependencyParserME implements DependencyParser { + + private final MaxentModel model; + private final DependencyContextGenerator contextGenerator; + private final Transition[] transitions; + + /** + * Initializes a {@link DependencyParserME} from a {@link DependencyModel}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. + */ + public DependencyParserME(DependencyModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model.getParserModel(); + this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(this.model); + } + + /** + * Initializes a {@link DependencyParserME} with a raw transition model. + * + * @param model The transition classification model. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. + */ + public DependencyParserME(MaxentModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model; + this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(model); + } + + /** + * Decodes the outcome inventory once, so that decoding a sentence indexes it instead + * of parsing an outcome string per configuration and outcome. + * + * @param model The transition classification model. + * @return The transitions by outcome index. Never {@code null}. + * @throws IllegalArgumentException Thrown if an outcome does not decode to a + * transition, which means the model is not a dependency parser model. + */ + private static Transition[] decodeOutcomes(MaxentModel model) { + final Transition[] decoded = new Transition[model.getNumOutcomes()]; + for (int i = 0; i < decoded.length; i++) { + final String outcome = model.getOutcome(i); + try { + decoded[i] = Transition.decode(outcome); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("model outcome is not a transition: " + outcome, e); + } + } + return decoded; + } + + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + state.apply(bestApplicable(state, tokens, tags)); + } + return state.toGraph(); + } + + /** + * Scores all outcomes for the current configuration and picks the best transition that + * is applicable; inapplicable outcomes are passed over regardless of score. + */ + private Transition bestApplicable(ArcStandardState state, String[] tokens, String[] tags) { + final double[] probabilities = model.eval(contextGenerator.getContext(state, tokens, tags)); + Transition best = null; + double bestProbability = Double.NEGATIVE_INFINITY; + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] <= bestProbability) { + continue; + } + if (state.canApply(transitions[i])) { + best = transitions[i]; + bestProbability = probabilities[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + return best; + } + + /** + * Trains a greedy arc-standard parser model from dependency samples. + * + *

Non-projective samples have no arc-standard derivation and are skipped during + * event generation.

+ * + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. + * @param samples The training samples. Must not be {@code null}. + * @param parameters The {@link TrainingParameters}. Must not be {@code null} and must + * select an event model trainer. + * @return A trained {@link DependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or the + * configured trainer is not an event model trainer. + */ + public static DependencyModel train(String languageCode, + ObjectStream samples, TrainingParameters parameters) + throws IOException { + if (languageCode == null || samples == null || parameters == null) { + throw new IllegalArgumentException( + "languageCode, samples and parameters must not be null"); + } + final TrainerType trainerType = TrainerFactory.getTrainerType(parameters); + if (!TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { + throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); + } + final Map manifestInfoEntries = new HashMap<>(); + final EventTrainer trainer = + TrainerFactory.getEventTrainer(parameters, manifestInfoEntries); + final ObjectStream events = + new DependencyEventStream(samples, new DependencyContextGenerator()); + return new DependencyModel(languageCode, trainer.train(events), manifestInfoEntries); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java new file mode 100644 index 0000000000..29fb8e4c87 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The feature template of the feedforward parser: a fixed set of configuration + * positions whose words, tags, and arc labels are embedded and concatenated into the + * network input, following + * Chen and Manning (2014). + * + *

Positions: the top three stack and buffer items; the leftmost and rightmost + * dependents of the top two stack items; and the leftmost dependent of the leftmost + * dependent and rightmost dependent of the rightmost dependent of the top two stack + * items, capturing second-order structure. Words and tags are read for all positions, + * labels only for the dependent positions, whose relations are already assigned.

+ */ +final class FeedforwardContext { + + /** The number of positions whose word and tag are embedded. */ + static final int POSITIONS = 14; + + /** The number of dependent positions whose arc label is embedded. */ + static final int LABEL_POSITIONS = 8; + + /** The index of the first dependent position; the stack and buffer items precede it. */ + private static final int FIRST_DEPENDENT_POSITION = 6; + + private FeedforwardContext() { + // This class only exposes the static feature template and is never instantiated. + } + + /** + * Extracts the symbolic features of a configuration: {@link #POSITIONS} words, then + * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent positions + * yield {@code null} entries, which the vocabulary maps to its padding symbol. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The symbolic features, in the order described above. Never {@code null}. + */ + static String[] extract(ArcStandardState state, String[] tokens, String[] tags) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int[] positions = { + s0, s1, state.stack(2), + state.buffer(0), state.buffer(1), state.buffer(2), + leftmost(state, s0), rightmost(state, s0), + leftmost(state, s1), rightmost(state, s1), + leftmost(state, leftmost(state, s0)), rightmost(state, rightmost(state, s0)), + leftmost(state, leftmost(state, s1)), rightmost(state, rightmost(state, s1)) + }; + final String[] features = new String[2 * POSITIONS + LABEL_POSITIONS]; + for (int i = 0; i < POSITIONS; i++) { + features[i] = symbol(tokens, positions[i]); + features[POSITIONS + i] = symbol(tags, positions[i]); + } + for (int i = 0; i < LABEL_POSITIONS; i++) { + final int position = positions[FIRST_DEPENDENT_POSITION + i]; + features[2 * POSITIONS + i] = + position >= 0 ? state.assignedRelation(position) : null; + } + return features; + } + + /** The leftmost dependent of a position, or the absence marker for absent positions. */ + private static int leftmost(ArcStandardState state, int index) { + return index >= 0 ? state.leftmostDependent(index) : ArcStandardState.NONE; + } + + /** The rightmost dependent of a position, or the absence marker for absent positions. */ + private static int rightmost(ArcStandardState state, int index) { + return index >= 0 ? state.rightmostDependent(index) : ArcStandardState.NONE; + } + + /** The value at a position: the root symbol for the root, {@code null} when absent. */ + private static String symbol(String[] values, int index) { + if (index == ArcStandardState.ROOT) { + return FeedforwardDependencyModel.ROOT_SYMBOL; + } + return index == ArcStandardState.NONE ? null : values[index]; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java new file mode 100644 index 0000000000..dca28791b7 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; + +import opennlp.tools.util.StringUtil; + +/** + * The weights of the feedforward transition parser: embeddings for words, tags, and arc + * labels, one hidden layer with cube activation, and a transition output layer, stored + * in a plain versioned binary format with no serialization framework involved. The + * architecture follows + * Chen and Manning (2014). + * + *

This is the pure-Java neural tier: the network is executed with ordinary array + * arithmetic, so parsing needs no native runtime, and the same class scores + * configurations for training and decoding. Unknown words fall back to a learned + * unknown symbol; words are matched case-insensitively after + * {@link #normalize(String) normalization}.

+ * + *

An instance is immutable and safe to share between threads once it has been handed + * to a caller. {@link FeedforwardDependencyTrainer} fills the weights while building a + * model and before that model escapes, and + * {@link FeedforwardDependencyTrainer#refine refine} trains a copy rather than the model + * it is given, so no model a caller holds ever changes underneath it.

+ * + * @see FeedforwardDependencyParser + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +public class FeedforwardDependencyModel { + + private static final String MAGIC = "ONLP-FFDP-1"; + + /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point with a contextual lowering. */ + private static final int GREEK_CAPITAL_SIGMA = 0x03A3; + + /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the capital. */ + private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2'; + + /** The vocabulary key of every word, tag, or label the model has no embedding row for. */ + static final String UNKNOWN = "*UNK*"; + + /** The vocabulary key of a template position that does not exist in a configuration. */ + static final String ABSENT = "*NULL*"; + + /** The vocabulary key of the artificial root node. */ + static final String ROOT_SYMBOL = "*ROOT*"; + + /** The prefix marking a vocabulary key as one of the special symbols above. */ + static final String SPECIAL_SYMBOL_PREFIX = "*"; + + /** The lazy scoring cache; {@code null} until {@link #enableScoringCache()}. */ + private volatile ContributionCache cache; + + private final Map wordIds; + private final Map tagIds; + private final Map labelIds; + private final String[] transitions; + + private final int embeddingSize; + private final float[][] embeddings; + private final float[][] hiddenWeights; + private final float[] hiddenBias; + private final float[][] outputWeights; + private final float[] outputBias; + + FeedforwardDependencyModel(Map wordIds, Map tagIds, + Map labelIds, String[] transitions, int embeddingSize, + float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias, + float[][] outputWeights, float[] outputBias) { + this.wordIds = Map.copyOf(wordIds); + this.tagIds = Map.copyOf(tagIds); + this.labelIds = Map.copyOf(labelIds); + this.transitions = transitions; + this.embeddingSize = embeddingSize; + this.embeddings = embeddings; + this.hiddenWeights = hiddenWeights; + this.hiddenBias = hiddenBias; + this.outputWeights = outputWeights; + this.outputBias = outputBias; + } + + /** + * Scores every transition for a configuration described by embedding row indices. + * + * @param features The embedding rows of the configuration, as produced by + * {@link #featureIds(String[])}. Must not be {@code null}. + * @return One unnormalized score per transition, indexed like + * {@link #transitions()}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code features} is {@code null}. + */ + public double[] score(int[] features) { + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } + final int hidden = hiddenBias.length; + final double[] h = new double[hidden]; + for (int j = 0; j < hidden; j++) { + h[j] = hiddenBias[j]; + } + final ContributionCache cache = this.cache; + for (int f = 0; f < features.length; f++) { + final int row = features[f]; + final float[] contribution = cache == null ? null : cache.contribution(this, f, row); + if (contribution != null) { + for (int j = 0; j < hidden; j++) { + h[j] += contribution[j]; + } + } else { + final float[] embedding = embeddings[row]; + final int offset = f * embeddingSize; + for (int j = 0; j < hidden; j++) { + final float[] weights = hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < embeddingSize; d++) { + sum += weights[offset + d] * embedding[d]; + } + h[j] += sum; + } + } + } + for (int j = 0; j < hidden; j++) { + h[j] = h[j] * h[j] * h[j]; + } + final double[] scores = new double[transitions.length]; + for (int o = 0; o < scores.length; o++) { + final float[] row = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hidden; j++) { + sum += row[j] * h[j]; + } + scores[o] = sum; + } + return scores; + } + + /** + * Turns on the scoring cache: the hidden-layer contribution of a (template position, + * embedding row) pair is a fixed vector while the weights do not change, so it is + * computed once on first sight and afterwards added instead of being re-derived from + * the embedding on every configuration. + * + *

Cached contributions are rounded to floats once, so scores may differ from the + * uncached path in the last bits. The cache is bounded, safe for concurrent readers, + * and only valid on a model whose weights no longer change: training and refinement + * work on uncached copies, and {@link #copy()} never carries a cache over.

+ */ + void enableScoringCache() { + if (cache == null) { + cache = new ContributionCache(2 * FeedforwardContext.POSITIONS + + FeedforwardContext.LABEL_POSITIONS, embeddings.length); + } + } + + /** + * The bounded lazy contribution cache behind {@link #enableScoringCache()}: one + * slot per (template position, embedding row) pair, filled on first use. Filling is + * idempotent, so concurrent readers may compute a contribution twice but never see + * a partial one, and a shared budget bounds the total memory; pairs beyond the + * budget simply keep the direct path. + */ + private static final class ContributionCache { + + /** The most (position, row) pairs the cache will hold. At a hidden size of 400 + * this bounds the cache near 100 MB; typical models stay far below the cap + * because tag and label inventories are small and word usage is Zipf-shaped. */ + private static final int MAX_PAIRS = 65536; + + private final AtomicReferenceArray[] byPosition; + private final AtomicInteger remaining = new AtomicInteger(MAX_PAIRS); + + @SuppressWarnings("unchecked") + private ContributionCache(int positions, int rows) { + byPosition = new AtomicReferenceArray[positions]; + for (int f = 0; f < positions; f++) { + byPosition[f] = new AtomicReferenceArray<>(rows); + } + } + + /** + * Returns the cached hidden-layer contribution of one pair, computing and + * publishing it on first sight while the budget lasts. + * + * @param model The frozen model the contributions derive from. + * @param position The template position. + * @param row The embedding row at that position. + * @return The contribution vector, or {@code null} when the budget is spent and + * the pair is not cached. + */ + private float[] contribution(FeedforwardDependencyModel model, int position, int row) { + final AtomicReferenceArray slots = byPosition[position]; + float[] contribution = slots.get(row); + if (contribution != null) { + return contribution; + } + if (remaining.get() <= 0) { + return null; + } + final int hidden = model.hiddenBias.length; + final float[] embedding = model.embeddings[row]; + final int offset = position * model.embeddingSize; + contribution = new float[hidden]; + for (int j = 0; j < hidden; j++) { + final float[] weights = model.hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < model.embeddingSize; d++) { + sum += weights[offset + d] * embedding[d]; + } + contribution[j] = (float) sum; + } + if (slots.compareAndSet(row, null, contribution)) { + remaining.decrementAndGet(); + } else { + contribution = slots.get(row); + } + return contribution; + } + } + + /** + * Maps the symbolic features of {@link FeedforwardContext} onto embedding rows. + * + * @param symbols The symbolic features. Must not be {@code null}. + * @return The embedding row per feature. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code symbols} is {@code null}. + */ + public int[] featureIds(String[] symbols) { + if (symbols == null) { + throw new IllegalArgumentException("symbols must not be null"); + } + final int[] ids = new int[symbols.length]; + for (int i = 0; i < FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(wordIds, normalize(symbols[i])); + } + for (int i = FeedforwardContext.POSITIONS; i < 2 * FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(tagIds, symbols[i]); + } + for (int i = 2 * FeedforwardContext.POSITIONS; i < symbols.length; i++) { + ids[i] = lookup(labelIds, symbols[i]); + } + return ids; + } + + /** + * @return The transition outcome strings by output index. Never {@code null}. + */ + public String[] transitions() { + return transitions.clone(); + } + + /** + * Lowercases a word symbol; special symbols and absences pass through. + * + *

Case is mapped per code point through UnicodeData, the same mapping as + * {@link StringUtil#toLowerCase(CharSequence)}, with one contextual rule on top: a + * Greek capital sigma preceded by a letter and not followed by one lowercases to + * the final form U+03C2, the Final_Sigma condition of the Unicode + * SpecialCasing + * file restricted to a single token. Natural lowercase Greek text, and with it + * every vocabulary key derived from a treebank, spells a word-final sigma that + * way, so without the rule an uppercase Greek word would normalize to a spelling + * the vocabulary never contains. A word that is already lowercase, the common + * case at parse time, is returned unchanged without allocating.

+ * + * @param word The word to normalize. May be {@code null}. + * @return The vocabulary key of {@code word}, or {@code null} if {@code word} is + * {@code null}. + */ + static String normalize(String word) { + if (word == null) { + return null; + } + if (word.startsWith(SPECIAL_SYMBOL_PREFIX)) { + return word; + } + int i = 0; + while (i < word.length()) { + final int cp = word.codePointAt(i); + if (Character.toLowerCase(cp) != cp) { + break; + } + i += Character.charCount(cp); + } + if (i == word.length()) { + return word; + } + final StringBuilder lowered = new StringBuilder(word.length()); + lowered.append(word, 0, i); + while (i < word.length()) { + final int cp = word.codePointAt(i); + final int width = Character.charCount(cp); + if (cp == GREEK_CAPITAL_SIGMA && i > 0 + && Character.isLetter(word.codePointBefore(i)) + && (i + width >= word.length() + || !Character.isLetter(word.codePointAt(i + width)))) { + lowered.append(GREEK_SMALL_FINAL_SIGMA); + } else { + lowered.appendCodePoint(Character.toLowerCase(cp)); + } + i += width; + } + return lowered.toString(); + } + + /** + * Resolves a symbol to its embedding row: absences map to {@link #ABSENT}, symbols + * without a row of their own fall back to {@link #UNKNOWN}. + * + * @param ids The vocabulary to resolve against. + * @param symbol The symbol to resolve, or {@code null} for an absent position. + * @return The embedding row of the symbol or of its fallback. + */ + private static int lookup(Map ids, String symbol) { + Integer id = ids.get(symbol == null ? ABSENT : symbol); + if (id == null) { + id = ids.get(UNKNOWN); + } + if (id == null) { + throw new IllegalStateException("vocabulary has no " + UNKNOWN + " row to fall back on"); + } + return id; + } + + /** + * Writes the model in the versioned binary format. + * + * @param out The stream to write to. Must not be {@code null}. Not closed. + * @throws IOException Thrown if writing fails. + */ + public void serialize(OutputStream out) throws IOException { + if (out == null) { + throw new IllegalArgumentException("out must not be null"); + } + final DataOutputStream data = new DataOutputStream(new BufferedOutputStream(out)); + data.writeUTF(MAGIC); + writeVocabulary(data, wordIds); + writeVocabulary(data, tagIds); + writeVocabulary(data, labelIds); + data.writeInt(transitions.length); + for (final String transition : transitions) { + data.writeUTF(transition); + } + data.writeInt(embeddingSize); + writeMatrix(data, embeddings); + writeMatrix(data, hiddenWeights); + writeVector(data, hiddenBias); + writeMatrix(data, outputWeights); + writeVector(data, outputBias); + data.flush(); + } + + /** + * Loads a model from the versioned binary format. + * + * @param in The stream to read from. Must not be {@code null}. Not closed. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is not this format. + */ + public static FeedforwardDependencyModel load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final DataInputStream data = new DataInputStream(new BufferedInputStream(in)); + final String magic = data.readUTF(); + if (!MAGIC.equals(magic)) { + throw new IOException("not a feedforward dependency model: " + magic); + } + final Map wordIds = readVocabulary(data); + final Map tagIds = readVocabulary(data); + final Map labelIds = readVocabulary(data); + final String[] transitions = new String[data.readInt()]; + for (int i = 0; i < transitions.length; i++) { + transitions[i] = data.readUTF(); + } + final int embeddingSize = data.readInt(); + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions, + embeddingSize, readMatrix(data), readMatrix(data), readVector(data), + readMatrix(data), readVector(data)); + } + + /** + * Loads a model from a file. + * + * @param path The file to read. Must not be {@code null}. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is not this format. + */ + public static FeedforwardDependencyModel load(Path path) throws IOException { + if (path == null) { + throw new IllegalArgumentException("path must not be null"); + } + try (InputStream in = Files.newInputStream(path)) { + return load(in); + } + } + + /** + * Writes one vocabulary as its size followed by (symbol, id) pairs. + */ + private static void writeVocabulary(DataOutputStream data, Map ids) + throws IOException { + data.writeInt(ids.size()); + // Entries are written in ascending id order: the iteration order of the immutable + // maps is salted per JVM launch, and serializing the same model must produce the + // same bytes on every run. + final List> entries = new ArrayList<>(ids.entrySet()); + entries.sort(Map.Entry.comparingByValue()); + for (final Map.Entry entry : entries) { + data.writeUTF(entry.getKey()); + data.writeInt(entry.getValue()); + } + } + + /** + * Reads one vocabulary written by {@link #writeVocabulary}. + */ + private static Map readVocabulary(DataInputStream data) + throws IOException { + final int size = data.readInt(); + final Map ids = new HashMap<>(size * 2); + for (int i = 0; i < size; i++) { + final String symbol = data.readUTF(); + ids.put(symbol, data.readInt()); + } + return ids; + } + + /** + * Writes a rectangular matrix as its dimensions followed by its values in row order. + */ + private static void writeMatrix(DataOutputStream data, float[][] matrix) + throws IOException { + data.writeInt(matrix.length); + data.writeInt(matrix.length == 0 ? 0 : matrix[0].length); + for (final float[] row : matrix) { + for (final float value : row) { + data.writeFloat(value); + } + } + } + + /** + * Reads a matrix written by {@link #writeMatrix}. + */ + private static float[][] readMatrix(DataInputStream data) throws IOException { + final int rows = data.readInt(); + final int columns = data.readInt(); + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = data.readFloat(); + } + } + return matrix; + } + + /** + * Writes a vector as its length followed by its values. + */ + private static void writeVector(DataOutputStream data, float[] vector) throws IOException { + data.writeInt(vector.length); + for (final float value : vector) { + data.writeFloat(value); + } + } + + /** + * Reads a vector written by {@link #writeVector}. + */ + private static float[] readVector(DataInputStream data) throws IOException { + final float[] vector = new float[data.readInt()]; + for (int i = 0; i < vector.length; i++) { + vector[i] = data.readFloat(); + } + return vector; + } + + /** + * Creates an independent copy of this model: the weights and the transition + * inventory array are deep-copied, and the vocabularies are shared because their + * maps are immutable. + * + *

This lets a training pass update the copy without ever writing to a model a + * caller already holds, which is what keeps the immutability this class documents + * true.

+ * + * @return A copy of this model sharing no mutable state with it. Never {@code null}. + */ + FeedforwardDependencyModel copy() { + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions.clone(), + embeddingSize, copyOf(embeddings), copyOf(hiddenWeights), hiddenBias.clone(), + copyOf(outputWeights), outputBias.clone()); + } + + /** + * Deep-copies a matrix, row by row. + */ + private static float[][] copyOf(float[][] matrix) { + final float[][] copy = new float[matrix.length][]; + for (int r = 0; r < matrix.length; r++) { + copy[r] = matrix[r].clone(); + } + return copy; + } + + /** + * @return The immutable map from a normalized word to its embedding row. Never {@code null}. + */ + Map wordIds() { + return wordIds; + } + + /** + * @return The immutable map from a tag to its embedding row. Never {@code null}. + */ + Map tagIds() { + return tagIds; + } + + /** + * @return The immutable map from an arc label to its embedding row. Never {@code null}. + */ + Map labelIds() { + return labelIds; + } + + /** + * @return The width of one embedding row. + */ + int embeddingSize() { + return embeddingSize; + } + + /** + * @return The live embedding matrix, one row per vocabulary entry, not a copy: the + * trainer writes its updates into it. Never {@code null}. + */ + float[][] embeddings() { + return embeddings; + } + + /** + * @return The live hidden layer weights, not a copy. Never {@code null}. + */ + float[][] hiddenWeights() { + return hiddenWeights; + } + + /** + * @return The live hidden layer bias, not a copy. Never {@code null}. + */ + float[] hiddenBias() { + return hiddenBias; + } + + /** + * @return The live output layer weights, one row per transition, not a copy. Never + * {@code null}. + */ + float[][] outputWeights() { + return outputWeights; + } + + /** + * @return The live output layer bias, one entry per transition, not a copy. Never + * {@code null}. + */ + float[] outputBias() { + return outputBias; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java new file mode 100644 index 0000000000..9cd96699cf --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** + * The pure-Java neural {@link DependencyParser}: an arc-standard decoder over the + * {@link FeedforwardDependencyModel}, greedy by default and beamed when constructed + * with a beam size above one. + * + *

With a beam, the decoder keeps the highest scoring transition sequences side by + * side, scored by summed log-probabilities, so a single locally attractive but globally + * wrong transition can still be recovered while the correct parse remains inside the + * beam. Every complete arc-standard derivation of a sentence has the same length, which + * keeps the summed scores comparable without length normalization.

+ * + *

Inference is ordinary array arithmetic with no native runtime involved, so this + * parser deploys exactly like the classical one while scoring configurations with + * learned dense representations instead of sparse feature conjunctions.

+ * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +public class FeedforwardDependencyParser implements DependencyParser { + + private final FeedforwardDependencyModel model; + private final Transition[] transitions; + private final int beamSize; + + /** + * Initializes a greedy {@link FeedforwardDependencyParser}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model) { + this(model, 1); + } + + /** + * Initializes a {@link FeedforwardDependencyParser} with a beam. + * + * @param model The model to parse with. Must not be {@code null}. + * @param beamSize The number of transition sequences to keep side by side. Must be + * greater than zero; {@code 1} decodes greedily. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}, + * {@code beamSize} is not positive, or an outcome of the model does not + * decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model, int beamSize) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + if (beamSize < 1) { + throw new IllegalArgumentException("beamSize must be positive: " + beamSize); + } + this.model = model; + this.beamSize = beamSize; + // A parser only ever reads a frozen model, so the scoring cache is safe to turn + // on here; training and refinement work on uncached copies. + model.enableScoringCache(); + final String[] outcomes = model.transitions(); + this.transitions = new Transition[outcomes.length]; + for (int i = 0; i < outcomes.length; i++) { + transitions[i] = Transition.decode(outcomes[i]); + } + } + + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + if (beamSize == 1) { + return greedyParse(tokens, tags); + } + return beamParse(tokens, tags); + } + + /** + * Decodes greedily: the highest scoring applicable transition wins each step. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + */ + private DependencyGraph greedyParse(String[] tokens, String[] tags) { + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + final double[] scores = model.score( + model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + Transition best = null; + double bestScore = Double.NEGATIVE_INFINITY; + for (int i = 0; i < scores.length; i++) { + if (scores[i] > bestScore && state.canApply(transitions[i])) { + best = transitions[i]; + bestScore = scores[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + state.apply(best); + } + return state.toGraph(); + } + + /** One search alternative: a configuration, its summed log-probability score, and the + * transition that would advance it, {@code null} once complete. */ + private record Alternative(ArcStandardState state, double score, Transition next) { + } + + /** + * Decodes with a beam: the {@code beamSize} best transition sequences advance side by + * side and the best complete one wins. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + */ + private DependencyGraph beamParse(String[] tokens, String[] tags) { + List beam = + List.of(new Alternative(new ArcStandardState(tokens.length), 0.0, null)); + while (true) { + boolean advanced = false; + final List expansions = new ArrayList<>(); + for (final Alternative alternative : beam) { + if (alternative.state().isTerminal()) { + expansions.add(alternative); + continue; + } + advanced = true; + final double[] logProbabilities = logSoftmax(model.score( + model.featureIds(FeedforwardContext.extract(alternative.state(), tokens, tags)))); + for (int i = 0; i < logProbabilities.length; i++) { + if (alternative.state().canApply(transitions[i])) { + expansions.add(new Alternative(alternative.state(), + alternative.score() + logProbabilities[i], transitions[i])); + } + } + } + if (!advanced) { + break; + } + expansions.sort(Comparator.comparingDouble(Alternative::score).reversed()); + final List survivors = + new ArrayList<>(Math.min(beamSize, expansions.size())); + for (int i = 0; i < expansions.size() && survivors.size() < beamSize; i++) { + final Alternative expansion = expansions.get(i); + if (expansion.next() == null) { + survivors.add(expansion); + } else { + final ArcStandardState state = expansion.state().copy(); + state.apply(expansion.next()); + survivors.add(new Alternative(state, expansion.score(), null)); + } + } + if (survivors.isEmpty()) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in the beam"); + } + beam = survivors; + } + return beam.get(0).state().toGraph(); + } + + /** + * Normalizes raw transition scores to log-probabilities. + * + * @param scores The raw output scores. + * @return The log-softmax of {@code scores}. Never {@code null}. + */ + private double[] logSoftmax(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = max + Math.log(sum); + final double[] logProbabilities = new double[scores.length]; + for (int i = 0; i < scores.length; i++) { + logProbabilities[i] = scores[i] - logSum; + } + return logProbabilities; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java new file mode 100644 index 0000000000..7f7e0c07ed --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -0,0 +1,950 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Function; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.util.ObjectStream; + +/** + * Trains the {@link FeedforwardDependencyModel} entirely in Java: oracle-derived + * transition examples, minibatch AdaGrad over a softmax cross-entropy loss, cube + * activation, and inverted dropout on the hidden layer, the training recipe of + * Chen and Manning (2014). No external + * training framework is involved, so the whole neural tier, training and inference, is + * plain array arithmetic inside the JVM. + * + *

Words below the frequency cutoff share a learned unknown embedding; absent + * template positions share a learned padding embedding. Non-projective samples have no + * arc-standard derivation and are skipped. Training is deterministic for a fixed + * {@link Settings#seed()}.

+ * + * @since 3.0.0 + */ +public final class FeedforwardDependencyTrainer { + + private static final Logger logger = + LoggerFactory.getLogger(FeedforwardDependencyTrainer.class); + + private static final double ADAGRAD_EPSILON = 1e-6; + + private FeedforwardDependencyTrainer() { + // This class only exposes static training methods and is never instantiated. + } + + /** + * The training hyperparameters. + * + * @param embeddingSize The embedding dimensionality. Must be positive. + * @param hiddenSize The hidden layer width. Must be positive. + * @param epochs The number of passes over the examples. Must be positive. + * @param batchSize The minibatch size. Must be positive. + * @param learningRate The AdaGrad step size. Must be positive. + * @param l2 The L2 penalty applied to the dense weights. Must not be negative. + * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}. + * @param wordCutoff The minimum frequency for a word to get its own embedding. Must + * not be negative. + * @param seed The random seed making a run reproducible. + */ + public record Settings(int embeddingSize, int hiddenSize, int epochs, int batchSize, + double learningRate, double l2, double dropout, int wordCutoff, long seed) { + + /** + * Validates the hyperparameters. + * + * @throws IllegalArgumentException Thrown if a value is outside its documented + * range. + */ + public Settings { + if (embeddingSize <= 0 || hiddenSize <= 0 || epochs <= 0 || batchSize <= 0) { + throw new IllegalArgumentException("sizes, epochs and batch must be positive"); + } + if (learningRate <= 0.0 || l2 < 0.0) { + throw new IllegalArgumentException("learningRate must be positive, l2 not negative"); + } + if (!(dropout >= 0.0 && dropout < 1.0)) { + throw new IllegalArgumentException("dropout must be in [0, 1): " + dropout); + } + if (wordCutoff < 0) { + throw new IllegalArgumentException("wordCutoff must not be negative"); + } + } + + /** + * @return The default hyperparameters. Never {@code null}. + */ + public static Settings defaults() { + return new Settings(50, 200, 10, 256, 0.02, 1e-8, 0.5, 2, 17L); + } + } + + /** + * Trains a model from dependency samples. + * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or no + * trainable example can be derived from the samples. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings) throws IOException { + return train(samples, settings, null); + } + + /** + * Trains a model from dependency samples, seeding word embeddings from a pretrained + * source. + * + *

The provider is consulted once per vocabulary word during initialization; words + * it returns {@code null} for keep their random initialization, and all embeddings + * remain trainable afterwards. The pretrained source is a training-time ingredient + * only: the learned embeddings ship inside the model, so parsing carries no + * dependency on the source.

+ * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @param pretrained Maps a normalized word to its pretrained vector of exactly + * {@link Settings#embeddingSize()} dimensions, or {@code null} for + * unknown words. May be {@code null} to disable seeding. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if {@code samples} or {@code settings} is + * {@code null}, no trainable example can be derived, or a pretrained vector + * has the wrong dimensionality. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings, Function pretrained) + throws IOException { + if (samples == null || settings == null) { + throw new IllegalArgumentException("samples and settings must not be null"); + } + final List corpus = readAll(samples); + final FeedforwardDependencyModel model = initialize(corpus, settings); + if (pretrained != null) { + seed(model, pretrained, settings); + } + final List featureList = new ArrayList<>(); + final List goldList = new ArrayList<>(); + collectExamples(corpus, model, featureList, goldList); + if (featureList.isEmpty()) { + throw new IllegalArgumentException("no trainable examples in the samples"); + } + optimize(model, featureList, goldList, settings); + return model; + } + + /** + * Fine-tunes a locally trained model globally: sentences are decoded with a beam, the + * gold derivation is tracked through it, and the moment the gold prefix falls out of + * the beam an early update, in the sense of + * Collins and Roark (2004), pushes + * the model toward keeping it. The loss is a conditional likelihood over the beam's + * candidate paths, scored exactly like the beamed parser scores them, summed + * log-probabilities, so training optimizes the quantity decoding uses. + * + *

The refined weights are a copy: {@code model} itself is never written to, so a + * model already being parsed with, possibly by several threads, keeps behaving exactly + * as before while a refined successor is trained from it. The copy is updated with + * per-sentence AdaGrad steps and no dropout; {@link Settings#epochs()} counts the + * refinement passes. Refinement is deterministic for a fixed {@link Settings#seed()}. + * Parse afterwards with the same beam size.

+ * + *

The transition inventory comes from {@code model} and is not extended, because + * its size is the width of the trained output layer. A refinement corpus using a + * relation label the original training set lacked is therefore rejected rather than + * silently ignored.

+ * + * @param model The locally trained model to refine. Left untouched. Must not be + * {@code null}. + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters; {@code epochs}, {@code learningRate}, + * {@code l2}, and {@code seed} apply. Must not be {@code null}. + * @param beamSize The beam width to track the gold derivation in. Must be at least 2. + * @return A new refined model, distinct from {@code model}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code beamSize} is below 2, no trainable sample can be derived, or a sample + * requires a transition {@code model} does not know. + */ + public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model, + ObjectStream samples, Settings settings, int beamSize) + throws IOException { + if (model == null || samples == null || settings == null) { + throw new IllegalArgumentException("model, samples and settings must not be null"); + } + if (beamSize < 2) { + throw new IllegalArgumentException("beamSize must be at least 2: " + beamSize); + } + final List corpus = readAll(samples); + final String[] outcomes = model.transitions(); + final Map transitionIds = new HashMap<>(); + final Transition[] transitions = new Transition[outcomes.length]; + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(outcomes[i], i); + transitions[i] = Transition.decode(outcomes[i]); + } + + final List trainable = new ArrayList<>(); + final List oracles = new ArrayList<>(); + for (final DependencySample s : corpus) { + final List oracle; + try { + oracle = ArcStandardOracle.transitions(s.getGraph()); + } catch (IllegalArgumentException e) { + continue; + } + final int[] encoded = new int[oracle.size()]; + for (int i = 0; i < encoded.length; i++) { + final String outcome = oracle.get(i).encode(); + final Integer id = transitionIds.get(outcome); + if (id == null) { + // The outcome space was fixed by the original training set, so a relation + // label it never saw has no output unit to push probability onto. + throw new IllegalArgumentException( + "unknown transition in the refinement samples: " + outcome); + } + encoded[i] = id; + } + trainable.add(s); + oracles.add(encoded); + } + if (trainable.isEmpty()) { + throw new IllegalArgumentException("no trainable samples for refinement"); + } + + final FeedforwardDependencyModel refined = model.copy(); + final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings); + final Random random = new Random(settings.seed()); + final int[] order = new int[trainable.size()]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + int updates = 0; + for (final int index : order) { + final double sentenceLoss = optimizer.refineSentence(trainable.get(index), + oracles.get(index), transitions, beamSize); + if (sentenceLoss >= 0.0) { + loss += sentenceLoss; + updates++; + } + } + logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch, + loss / Math.max(updates, 1), updates, System.currentTimeMillis() - epochStart); + } + return refined; + } + + /** One candidate path in the refinement beam: the parent link forms the history. */ + private static final class BeamNode { + private final BeamNode parent; + private final int[] features; + private final int transition; + private final double score; + private final boolean gold; + private ArcStandardState state; + + /** Extends {@code parent} by one transition; the start node passes {@code null}. */ + private BeamNode(BeamNode parent, int[] features, int transition, double score, + boolean gold) { + this.parent = parent; + this.features = features; + this.transition = transition; + this.score = score; + this.gold = gold; + } + } + + /** The forward, backward, and AdaGrad state for global refinement. */ + private static final class GlobalOptimizer { + private final FeedforwardDependencyModel model; + private final Settings settings; + private final int embeddingSize; + private final int hiddenSize; + private final int outputSize; + private final int inputSize; + + private final double[][] embeddingAccumulator; + private final double[][] hiddenAccumulator; + private final double[] hiddenBiasAccumulator; + private final double[][] outputAccumulator; + private final double[] outputBiasAccumulator; + + private final double[][] hiddenGradient; + private final double[] hiddenBiasGradient; + private final double[][] outputGradient; + private final double[] outputBiasGradient; + private final Map embeddingGradients = new HashMap<>(); + + private final double[] x; + private final double[] pre; + private final double[] hidden; + private final double[] probabilities; + private final double[] hiddenDelta; + private final double[] inputDelta; + + /** Sizes the accumulators and scratch buffers for one refinement run over {@code model}. */ + private GlobalOptimizer(FeedforwardDependencyModel model, Settings settings) { + this.model = model; + this.settings = settings; + this.embeddingSize = model.embeddings()[0].length; + this.hiddenSize = model.hiddenBias().length; + this.outputSize = model.outputBias().length; + this.inputSize = + (2 * FeedforwardContext.POSITIONS + FeedforwardContext.LABEL_POSITIONS) + * embeddingSize; + this.embeddingAccumulator = + new double[model.embeddings().length][embeddingSize]; + this.hiddenAccumulator = new double[hiddenSize][inputSize]; + this.hiddenBiasAccumulator = new double[hiddenSize]; + this.outputAccumulator = new double[outputSize][hiddenSize]; + this.outputBiasAccumulator = new double[outputSize]; + this.hiddenGradient = new double[hiddenSize][inputSize]; + this.hiddenBiasGradient = new double[hiddenSize]; + this.outputGradient = new double[outputSize][hiddenSize]; + this.outputBiasGradient = new double[outputSize]; + this.x = new double[inputSize]; + this.pre = new double[hiddenSize]; + this.hidden = new double[hiddenSize]; + this.probabilities = new double[outputSize]; + this.hiddenDelta = new double[hiddenSize]; + this.inputDelta = new double[inputSize]; + } + + /** + * Decodes one sentence with the beam, updating on the early-update point or the + * final beam. + * + * @param sample The sentence. + * @param oracle The gold transition indexes. + * @param transitions The decoded transition inventory. + * @param beamSize The beam width. + * @return The sentence loss, or {@code -1} when the sentence produced no update. + */ + private double refineSentence(DependencySample sample, int[] oracle, + Transition[] transitions, int beamSize) { + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + final BeamNode root = new BeamNode(null, null, -1, 0.0, true); + root.state = new ArcStandardState(tokens.length); + List beam = List.of(root); + + for (int step = 0; step < oracle.length; step++) { + final List expansions = new ArrayList<>(); + BeamNode goldChild = null; + for (final BeamNode node : beam) { + final int[] features = + model.featureIds(FeedforwardContext.extract(node.state, tokens, tags)); + forward(features); + logSoftmaxInPlace(probabilities); + for (int i = 0; i < outputSize; i++) { + if (node.state.canApply(transitions[i])) { + final boolean goldNext = node.gold && i == oracle[step]; + final BeamNode child = new BeamNode(node, features, i, + node.score + probabilities[i], goldNext); + expansions.add(child); + if (goldNext) { + goldChild = child; + } + } + } + } + expansions.sort((a, b) -> Double.compare(b.score, a.score)); + final List survivors = + new ArrayList<>(expansions.subList(0, Math.min(beamSize, expansions.size()))); + boolean goldSurvives = false; + for (final BeamNode survivor : survivors) { + if (survivor.gold) { + goldSurvives = true; + break; + } + } + if (!goldSurvives) { + if (goldChild == null) { + // The gold transition was not applicable in the gold configuration, so + // this sentence yields no update. + return -1.0; + } + survivors.add(goldChild); + return updateFromCandidates(survivors); + } + if (step == oracle.length - 1) { + return updateFromCandidates(survivors); + } + for (final BeamNode survivor : survivors) { + survivor.state = survivor.parent.state.copy(); + survivor.state.apply(transitions[survivor.transition]); + } + beam = survivors; + } + return -1.0; + } + + /** Applies the conditional-likelihood update over the candidate paths. */ + private double updateFromCandidates(List candidates) { + double max = Double.NEGATIVE_INFINITY; + double goldScore = Double.NEGATIVE_INFINITY; + for (final BeamNode candidate : candidates) { + max = Math.max(max, candidate.score); + if (candidate.gold) { + goldScore = candidate.score; + } + } + double normalizer = 0.0; + for (final BeamNode candidate : candidates) { + normalizer += Math.exp(candidate.score - max); + } + final double logNormalizer = max + Math.log(normalizer); + + zero(hiddenGradient); + Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + for (final BeamNode candidate : candidates) { + final double weight = Math.exp(candidate.score - logNormalizer) + - (candidate.gold ? 1.0 : 0.0); + if (weight == 0.0) { + continue; + } + for (BeamNode node = candidate; node.parent != null; node = node.parent) { + backward(node.features, node.transition, weight); + } + } + update(model.hiddenWeights(), hiddenGradient, hiddenAccumulator, 1, settings); + updateVector(model.hiddenBias(), hiddenBiasGradient, hiddenBiasAccumulator, 1, + settings); + update(model.outputWeights(), outputGradient, outputAccumulator, 1, settings); + updateVector(model.outputBias(), outputBiasGradient, outputBiasAccumulator, 1, + settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = model.embeddings()[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d]; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + return logNormalizer - goldScore; + } + + /** Computes hidden activations and raw output scores for one feature vector. */ + private void forward(int[] features) { + final float[][] embeddings = model.embeddings(); + for (int f = 0; f < features.length; f++) { + final float[] embedding = embeddings[features[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + for (int j = 0; j < hiddenSize; j++) { + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum; + } + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + } + } + + /** + * Accumulates gradients for one decoded step: the weighted difference between the + * step's softmax and its chosen transition. + * + * @param features The step's input features. + * @param chosen The transition the path took at this step. + * @param weight The path's weight in the candidate distribution. + */ + private void backward(int[] features, int chosen, double weight) { + forward(features); + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + max = Math.max(max, probabilities[o]); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); + final float[][] outputWeights = model.outputWeights(); + for (int o = 0; o < outputSize; o++) { + // dL/dlogit for a path's step under the conditional likelihood: the path weight + // times how the step's log-probability responds to this logit + final double delta = + weight * ((o == chosen ? 1.0 : 0.0) - probabilities[o] / normalizer); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + for (int j = 0; j < hiddenSize; j++) { + final double preDelta = hiddenDelta[j] * 3.0 * pre[j] * pre[j]; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < features.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(features[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + /** Turns raw scores into log-probabilities in place. */ + private void logSoftmaxInPlace(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = max + Math.log(sum); + for (int i = 0; i < scores.length; i++) { + scores[i] -= logSum; + } + } + } + + /** + * Reads a sample stream into memory; both trainers pass over the corpus repeatedly. + * + * @param samples The stream to drain. + * @return All samples in stream order. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + */ + private static List readAll(ObjectStream samples) + throws IOException { + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + return corpus; + } + + /** Overwrites the random word rows with pretrained vectors where available. */ + private static void seed(FeedforwardDependencyModel model, + Function pretrained, Settings settings) { + int seeded = 0; + for (final Map.Entry entry : model.wordIds().entrySet()) { + if (entry.getKey().startsWith(FeedforwardDependencyModel.SPECIAL_SYMBOL_PREFIX)) { + // The special unknown, padding, and root symbols have no pretrained + // counterpart, so they keep their random initialization. + continue; + } + final float[] vector = pretrained.apply(entry.getKey()); + if (vector == null) { + continue; + } + if (vector.length != settings.embeddingSize()) { + throw new IllegalArgumentException("pretrained vector for '" + entry.getKey() + + "' has " + vector.length + " dimensions, expected " + + settings.embeddingSize()); + } + System.arraycopy(vector, 0, model.embeddings()[entry.getValue()], 0, vector.length); + seeded++; + } + logger.info("seeded {} of {} word embeddings from the pretrained source", seeded, + model.wordIds().size()); + } + + /** Builds the vocabularies and randomly initialized weights. */ + private static FeedforwardDependencyModel initialize(List corpus, + Settings settings) { + final Map wordCounts = new HashMap<>(); + final Map tagIds = new HashMap<>(); + final Map labelIds = new HashMap<>(); + final Map transitionIds = new HashMap<>(); + for (final DependencySample s : corpus) { + for (final String token : s.getTokens()) { + wordCounts.merge(FeedforwardDependencyModel.normalize(token), 1, Integer::sum); + } + for (final String tag : s.getTags()) { + tagIds.putIfAbsent(tag, 0); + } + final DependencyGraph graph = s.getGraph(); + for (int i = 0; i < graph.size(); i++) { + labelIds.putIfAbsent(graph.relationOf(i), 0); + } + } + // The outcome space is the shift transition plus both arc directions for every + // relation label observed in the training data. + transitionIds.putIfAbsent(Transition.SHIFT.encode(), 0); + for (final String label : labelIds.keySet()) { + transitionIds.putIfAbsent(Transition.leftArc(label).encode(), 0); + transitionIds.putIfAbsent(Transition.rightArc(label).encode(), 0); + } + + int row = 0; + final Map wordIds = new HashMap<>(); + row = addSpecialSymbols(wordIds, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); + for (final Map.Entry entry : wordCounts.entrySet()) { + if (entry.getValue() >= settings.wordCutoff() && !wordIds.containsKey(entry.getKey())) { + wordIds.put(entry.getKey(), row++); + } + } + final Map tags = new HashMap<>(); + row = addSpecialSymbols(tags, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); + for (final String tag : tagIds.keySet()) { + tags.put(tag, row++); + } + final Map labels = new HashMap<>(); + row = addSpecialSymbols(labels, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT); + for (final String label : labelIds.keySet()) { + labels.put(label, row++); + } + + int transitionIndex = 0; + final String[] transitions = new String[transitionIds.size()]; + for (final String encoded : transitionIds.keySet()) { + transitions[transitionIndex] = encoded; + transitionIds.put(encoded, transitionIndex++); + } + + final Random random = new Random(settings.seed()); + final int inputSize = + (2 * FeedforwardContext.POSITIONS + FeedforwardContext.LABEL_POSITIONS) + * settings.embeddingSize(); + final float[][] embeddings = uniform(random, row, settings.embeddingSize(), 0.01); + final float[][] hiddenWeights = uniform(random, settings.hiddenSize(), inputSize, + Math.sqrt(6.0 / (inputSize + settings.hiddenSize()))); + final float[][] outputWeights = uniform(random, transitions.length, + settings.hiddenSize(), + Math.sqrt(6.0 / (settings.hiddenSize() + transitions.length))); + return new FeedforwardDependencyModel(wordIds, tags, labels, transitions, + settings.embeddingSize(), embeddings, hiddenWeights, + new float[settings.hiddenSize()], outputWeights, new float[transitions.length]); + } + + /** + * Assigns the next embedding rows to the special symbols of one vocabulary. + * + * @param ids The vocabulary to fill. + * @param row The next free embedding row. + * @param symbols The special symbols, in the order they take their rows. + * @return The next free embedding row after the symbols. + */ + private static int addSpecialSymbols(Map ids, int row, String... symbols) { + int next = row; + for (final String symbol : symbols) { + ids.put(symbol, next); + next++; + } + return next; + } + + /** Replays the oracle over every projective sample, emitting one example per step. */ + private static void collectExamples(List corpus, + FeedforwardDependencyModel model, List featureList, List goldList) { + final Map transitionIds = new HashMap<>(); + final String[] transitions = model.transitions(); + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(transitions[i], i); + } + int skipped = 0; + for (final DependencySample sample : corpus) { + final List oracle; + try { + oracle = ArcStandardOracle.transitions(sample.getGraph()); + } catch (IllegalArgumentException e) { + skipped++; + continue; + } + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : oracle) { + featureList.add(model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + goldList.add(transitionIds.get(transition.encode())); + state.apply(transition); + } + } + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + } + } + + /** Minibatch AdaGrad over softmax cross-entropy with cube activation and dropout. */ + private static void optimize(FeedforwardDependencyModel model, List featureList, + List goldList, Settings settings) { + final int exampleCount = featureList.size(); + final int[][] features = featureList.toArray(new int[0][]); + final int[] gold = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + gold[i] = goldList.get(i); + } + + final float[][] embeddings = model.embeddings(); + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + final int embeddingSize = settings.embeddingSize(); + final int hiddenSize = settings.hiddenSize(); + final int outputSize = outputBias.length; + final int inputSize = features[0].length * embeddingSize; + + final double[][] embeddingAccumulator = + new double[embeddings.length][embeddingSize]; + final double[][] hiddenAccumulator = new double[hiddenSize][inputSize]; + final double[] hiddenBiasAccumulator = new double[hiddenSize]; + final double[][] outputAccumulator = new double[outputSize][hiddenSize]; + final double[] outputBiasAccumulator = new double[outputSize]; + + final double[][] hiddenGradient = new double[hiddenSize][inputSize]; + final double[] hiddenBiasGradient = new double[hiddenSize]; + final double[][] outputGradient = new double[outputSize][hiddenSize]; + final double[] outputBiasGradient = new double[outputSize]; + final Map embeddingGradients = new HashMap<>(); + + final Random random = new Random(settings.seed()); + final int[] order = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + order[i] = i; + } + + final double keep = 1.0 - settings.dropout(); + final double[] x = new double[inputSize]; + final double[] pre = new double[hiddenSize]; + final double[] hidden = new double[hiddenSize]; + final boolean[] mask = new boolean[hiddenSize]; + final double[] probabilities = new double[outputSize]; + final double[] hiddenDelta = new double[hiddenSize]; + final double[] inputDelta = new double[inputSize]; + + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + for (int batchStart = 0; batchStart < exampleCount; + batchStart += settings.batchSize()) { + final int batchEnd = Math.min(batchStart + settings.batchSize(), exampleCount); + final int batch = batchEnd - batchStart; + zero(hiddenGradient); + Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + + for (int b = batchStart; b < batchEnd; b++) { + final int[] feats = features[order[b]]; + final int goldTransition = gold[order[b]]; + for (int f = 0; f < feats.length; f++) { + final float[] embedding = embeddings[feats[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + for (int j = 0; j < hiddenSize; j++) { + mask[j] = random.nextDouble() < keep; + if (!mask[j]) { + pre[j] = 0.0; + hidden[j] = 0.0; + continue; + } + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum / keep; + } + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + max = Math.max(max, sum); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + for (int o = 0; o < outputSize; o++) { + probabilities[o] /= normalizer; + } + loss -= Math.log(Math.max(probabilities[goldTransition], 1e-12)); + + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); + for (int o = 0; o < outputSize; o++) { + final double delta = probabilities[o] - (o == goldTransition ? 1.0 : 0.0); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + for (int j = 0; j < hiddenSize; j++) { + if (!mask[j]) { + continue; + } + final double preDelta = hiddenDelta[j] * 3.0 * pre[j] * pre[j] / keep; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < feats.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(feats[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + update(hiddenWeights, hiddenGradient, hiddenAccumulator, batch, settings); + updateVector(hiddenBias, hiddenBiasGradient, hiddenBiasAccumulator, batch, settings); + update(outputWeights, outputGradient, outputAccumulator, batch, settings); + updateVector(outputBias, outputBiasGradient, outputBiasAccumulator, batch, settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = embeddings[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d] / batch; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + } + logger.info("epoch {}: loss {} in {} ms", epoch, loss / exampleCount, + System.currentTimeMillis() - epochStart); + } + } + + /** One AdaGrad step on a weight matrix, with the L2 penalty folded into the gradient. */ + private static void update(float[][] weights, double[][] gradients, + double[][] accumulators, int batch, Settings settings) { + for (int r = 0; r < weights.length; r++) { + final float[] weightRow = weights[r]; + final double[] gradientRow = gradients[r]; + final double[] accumulatorRow = accumulators[r]; + for (int c = 0; c < weightRow.length; c++) { + final double gradient = gradientRow[c] / batch + settings.l2() * weightRow[c]; + accumulatorRow[c] += gradient * gradient; + weightRow[c] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[c]) + ADAGRAD_EPSILON); + } + } + } + + /** One AdaGrad step on a bias vector; biases carry no L2 penalty. */ + private static void updateVector(float[] weights, double[] gradients, + double[] accumulators, int batch, Settings settings) { + for (int i = 0; i < weights.length; i++) { + final double gradient = gradients[i] / batch; + accumulators[i] += gradient * gradient; + weights[i] -= settings.learningRate() * gradient + / (Math.sqrt(accumulators[i]) + ADAGRAD_EPSILON); + } + } + + /** A matrix drawn uniformly from {@code [-scale, scale]}. */ + private static float[][] uniform(Random random, int rows, int columns, double scale) { + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = (float) ((random.nextDouble() * 2.0 - 1.0) * scale); + } + } + return matrix; + } + + /** Fills a matrix with zeros. */ + private static void zero(double[][] matrix) { + for (final double[] row : matrix) { + Arrays.fill(row, 0.0); + } + } + + /** Fisher-Yates shuffle of the visit order. */ + private static void shuffle(int[] order, Random random) { + for (int i = order.length - 1; i > 0; i--) { + final int j = random.nextInt(i + 1); + final int swap = order[i]; + order[i] = order[j]; + order[j] = swap; + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java new file mode 100644 index 0000000000..8095e48102 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.StringUtil; + +/** + * One action of the arc-standard transition system: shift the next buffer token onto the + * stack, or attach one of the two topmost stack tokens to the other under a relation label. + * + *

A transition doubles as a classification outcome: {@link #encode()} renders it as the + * outcome string a model is trained on, and {@link #decode(String)} restores it.

+ * + * @param type The kind of action. Must not be {@code null}. + * @param label The relation label for arc actions, {@code null} for {@link Type#SHIFT}. + * + * @since 3.0.0 + */ +public record Transition(Type type, String label) { + + /** + * The kinds of arc-standard action. + */ + public enum Type { + /** Pushes the front of the buffer onto the stack. */ + SHIFT, + /** Attaches the second stack token to the top one and removes the second. */ + LEFT_ARC, + /** Attaches the top stack token to the second one and removes the top. */ + RIGHT_ARC + } + + /** The single shift transition; shifts carry no label. */ + public static final Transition SHIFT = new Transition(Type.SHIFT, null); + + private static final char SEPARATOR = ':'; + + /** + * Validates the pairing of type and label. + * + * @throws IllegalArgumentException Thrown if {@code type} is {@code null}, a shift + * carries a label, or an arc action has a {@code null} or blank label. + */ + public Transition { + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + if (type == Type.SHIFT) { + if (label != null) { + throw new IllegalArgumentException("a shift must not carry a label: " + label); + } + } else if (label == null || StringUtil.isBlank(label)) { + throw new IllegalArgumentException("an arc transition needs a relation label"); + } + } + + /** + * Creates a left-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#LEFT_ARC}. Never {@code null}. + */ + public static Transition leftArc(String label) { + return new Transition(Type.LEFT_ARC, label); + } + + /** + * Creates a right-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#RIGHT_ARC}. Never {@code null}. + */ + public static Transition rightArc(String label) { + return new Transition(Type.RIGHT_ARC, label); + } + + /** + * Renders the transition as a model outcome string, for example {@code SHIFT} or + * {@code LEFT_ARC:nsubj}. + * + * @return The outcome string. Never {@code null}. + */ + public String encode() { + return type == Type.SHIFT ? type.name() : type.name() + SEPARATOR + label; + } + + /** + * Restores a transition from a model outcome string produced by {@link #encode()}. + * + * @param outcome The outcome string. Must not be {@code null}. + * @return The decoded {@link Transition}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code outcome} is {@code null} or does not + * name a valid transition. + */ + public static Transition decode(String outcome) { + if (outcome == null) { + throw new IllegalArgumentException("outcome must not be null"); + } + if (Type.SHIFT.name().equals(outcome)) { + return SHIFT; + } + final int separator = outcome.indexOf(SEPARATOR); + if (separator < 0) { + throw new IllegalArgumentException("not a transition outcome: " + outcome); + } + final Type type; + try { + type = Type.valueOf(outcome.substring(0, separator)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("not a transition outcome: " + outcome, e); + } + return new Transition(type, outcome.substring(separator + 1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java new file mode 100644 index 0000000000..2c633798e5 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link Lemmatizer} to the document pipeline: reads {@link Layers#SENTENCES}, + * {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and provides {@link #LEMMAS}, one + * annotation per token on the token's span. + * + *

Each sentence is lemmatized separately, the way the lemmatizer contract expects its + * input, so lemmatization decisions never cross a sentence boundary. Token spans already + * refer to the original document text, so only the token and tag sequences handed to the + * lemmatizer are sliced per sentence; the produced lemma layer stays aligned with + * {@link Layers#TOKENS} by position.

+ * + * @since 3.0.0 + */ +public final class LemmatizerAnnotator implements DocumentAnnotator { + + /** + * The lemma layer. It is aligned with the token layer by position, and each annotation + * carries the lemma of its token on that token's span. + */ + public static final LayerKey LEMMAS = Layers.key("lemmas", String.class); + + private final Lemmatizer lemmatizer; + + /** + * Initializes the adapter. + * + * @param lemmatizer The lemmatizer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code lemmatizer} is {@code null}. + */ + public LemmatizerAnnotator(Lemmatizer lemmatizer) { + if (lemmatizer == null) { + throw new IllegalArgumentException("lemmatizer must not be null"); + } + this.lemmatizer = lemmatizer; + } + + /** + * Lemmatizes the document sentence by sentence and adds the {@link #LEMMAS} layer. + * + *

For every sentence, the tokens whose spans lie inside the sentence span are + * lemmatized as one sequence together with their tags, and each lemma is emitted on + * its token's span. The required layers must be present, but they may be empty: a + * document without sentences or tokens yields a present-but-empty lemma layer, and a + * sentence containing no tokens contributes nothing.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers and a + * {@link Layers#POS_TAGS} layer with exactly one tag per token, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link #LEMMAS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * sentence layer, the token layer, or the tag layer is absent, the tag layer + * does not have exactly one tag per token, a token lies outside every + * sentence, or the lemmatizer does not return one lemma per token of a + * sentence. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, + Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final List> layer = new ArrayList<>(tokens.size()); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final String[] posTags = new String[words.length]; + for (int i = 0; i < words.length; i++) { + posTags[i] = tags.get(first + i).value(); + } + final String[] lemmas = lemmatizer.lemmatize(words, posTags); + if (lemmas.length != words.length) { + throw new IllegalArgumentException("lemmatizer returned " + lemmas.length + + " lemmas for " + words.length + " tokens"); + } + for (int i = 0; i < words.length; i++) { + layer.add(new Annotation<>(tokens.get(first + i).span(), lemmas[i])); + } + }); + return document.with(LEMMAS, layer); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(LEMMAS); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java new file mode 100644 index 0000000000..43baa231e0 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.namefind; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link TokenNameFinder} to the document pipeline: reads + * {@link Layers#SENTENCES} and {@link Layers#TOKENS}, maps the finder's token-index + * spans to character spans on the original text, and provides {@link Layers#ENTITIES}. + * The entity type is carried as the annotation value; the annotation's span carries + * offsets only. + * + *

Each sentence's tokens are passed to {@link TokenNameFinder#find(String[])} as one + * sequence, the way the finder contract expects its input, so no mention can straddle a + * sentence boundary. The finder's adaptive data is cleared exactly once per call, as the + * {@link TokenNameFinder#clearAdaptiveData()} contract asks, whether annotation succeeds + * or fails, so no document can leak finder state into the next one.

+ * + *

Spans the finder returns without a type are recorded with the {@link #UNTYPED} + * entity type.

+ * + * @since 3.0.0 + */ +public final class NameFinderAnnotator implements DocumentAnnotator { + + /** + * The entity type recorded when the wrapped finder returns a span without a type. It + * is {@link NameSample#DEFAULT_TYPE}. Type-aware consumers should treat this label as + * an unknown type rather than as a distinct one, since it carries no information about + * what kind of entity was found. + */ + public static final String UNTYPED = NameSample.DEFAULT_TYPE; + + private final TokenNameFinder finder; + + /** + * Initializes the adapter. + * + * @param finder The name finder to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code finder} is {@code null}. + */ + public NameFinderAnnotator(TokenNameFinder finder) { + if (finder == null) { + throw new IllegalArgumentException("finder must not be null"); + } + this.finder = finder; + } + + /** + * Finds names sentence by sentence and adds the {@link Layers#ENTITIES} layer. + * + *

For every sentence, the tokens whose spans lie inside the sentence span are + * passed to the finder as one sequence, and each sentence-local mention is mapped + * through the sentence's first token position onto character spans of the original + * text. The required layers must be present, but they may be empty: a document + * without sentences or tokens yields a present-but-empty entity layer, and a sentence + * containing no tokens contributes nothing. A mention without a type is recorded with + * the type {@link #UNTYPED} as the annotation value.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link Layers#ENTITIES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * sentence layer or the token layer is absent, a token lies outside every + * sentence, or the finder returns a mention that is empty or whose token + * indices lie outside its sentence's tokens. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> entities = new ArrayList<>(); + // The adaptive data is cleared even when annotation fails, so a rejected document + // cannot leak finder state into the next one. + try { + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + // The finder indexes within the sentence; shifting by the sentence's first + // token position turns every mention boundary into a document-wide token + // index, whose token spans already refer to the original text. An empty + // mention is rejected with the out-of-bounds ones: it covers no token, so it + // has no character span. + for (final Span mention : finder.find(words)) { + if (mention.getStart() < 0 || mention.getEnd() > words.length + || mention.getStart() >= mention.getEnd()) { + throw new IllegalArgumentException("finder returned mention " + mention + + " outside the sentence's " + words.length + " tokens"); + } + final int start = tokens.get(first + mention.getStart()).span().getStart(); + final int end = tokens.get(first + mention.getEnd() - 1).span().getEnd(); + final String type = mention.getType() == null ? UNTYPED : mention.getType(); + entities.add(new Annotation<>(new Span(start, end), type)); + } + }); + } finally { + finder.clearAdaptiveData(); + } + return document.with(Layers.ENTITIES, entities); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(Layers.ENTITIES); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java new file mode 100644 index 0000000000..e38b2fdb22 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Adapts a constituency {@link Parser} to the document pipeline: reads + * {@link Layers#SENTENCES} and {@link Layers#TOKENS} and provides {@link #PHRASES}, one + * annotation per phrase node of each sentence's parse, carrying the phrase label and the + * span of its head token. + * + *

Each sentence is parsed from its tokens as one sequence. Every node above the + * part-of-speech level except the root becomes an annotation on the span from its first + * to its last token, in pre-order, so an enclosing phrase precedes the phrases it + * contains and phrases nest by span containment. Part-of-speech nodes are left to the + * {@link Layers#POS_TAGS} layer and token nodes to {@link Layers#TOKENS}. The head token + * is the one the parser's head rules select, so a consumer can read the head of a noun + * phrase without its own rules.

+ * + *

The adapter holds no per-call state; it is as thread-safe as the parser it + * wraps.

+ * + * @since 3.0.0 + */ +public final class ParserAnnotator implements DocumentAnnotator { + + /** + * One phrase of a constituency parse: its label, such as {@code NP} or {@code VP}, + * and the span of the token that heads it. The phrase's own span is the annotation's + * span. + * + * @param label The phrase label. Must not be {@code null} or blank. + * @param head The span of the head token in the document text. Must not be + * {@code null}. + * + * @since 3.0.0 + */ + public record Phrase(String label, Span head) { + + /** + * Validates the phrase. + * + * @throws IllegalArgumentException Thrown if {@code label} is {@code null} or + * blank, or {@code head} is {@code null}. + */ + public Phrase { + if (label == null || StringUtil.isBlank(label)) { + throw new IllegalArgumentException("label must not be null or blank"); + } + if (head == null) { + throw new IllegalArgumentException("head must not be null"); + } + } + } + + /** + * Parse phrases; each annotation covers one phrase and carries its {@link Phrase}, + * in pre-order of the parse tree. + */ + public static final LayerKey PHRASES = Layers.key("phrases", Phrase.class); + + private final Parser parser; + + /** + * Initializes the adapter. + * + * @param parser The parser to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public ParserAnnotator(Parser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + /** + * Parses the document sentence by sentence and adds the {@link #PHRASES} layer. + * + *

The required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty phrase layer.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link #PHRASES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, a + * required layer is absent, a token lies outside every sentence, or the + * parser returns a node outside the sentence's tokens. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> phrases = new ArrayList<>(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final Parse root = parser.parse(Parse.createFromTokens(words)); + if (root == null) { + throw new IllegalArgumentException("parser returned no parse"); + } + // The parse text is the tokens joined by single spaces, so a token's start in + // that text identifies its index. + final int[] starts = new int[words.length]; + for (int i = 1; i < words.length; i++) { + starts[i] = starts[i - 1] + words[i - 1].length() + 1; + } + final int length = starts[words.length - 1] + words[words.length - 1].length(); + for (final Parse child : root.getChildren()) { + collect(child, first, starts, length, tokens, phrases); + } + }); + return document.with(PHRASES, phrases); + } + + /** Emits a node and, in pre-order, every phrase node below it. */ + private void collect(Parse node, int first, int[] starts, int length, + List> tokens, List> phrases) { + if (node.isPosTag() || Parser.TOK_NODE.equals(node.getType())) { + return; + } + final int from = tokenIndex(starts, length, node.getSpan().getStart(), node); + final int to = tokenIndex(starts, length, node.getSpan().getEnd(), node); + final int head = node.getHeadIndex(); + if (head < 0 || head >= starts.length) { + throw new IllegalArgumentException("parser returned node " + node.getType() + + " with head " + head + " outside the sentence's " + starts.length + " tokens"); + } + phrases.add(new Annotation<>(new Span(tokens.get(first + from).span().getStart(), + tokens.get(first + to).span().getEnd()), + new Phrase(node.getType(), tokens.get(first + head).span()))); + for (final Parse child : node.getChildren()) { + collect(child, first, starts, length, tokens, phrases); + } + } + + /** + * Maps an offset in the parse text, the tokens joined by single spaces, to the index + * of the token it lies in or, for a span end, ends. + */ + private int tokenIndex(int[] starts, int length, int offset, Parse node) { + if (offset < 0 || offset > length) { + throw new IllegalArgumentException("parser returned node " + node.getType() + + " at " + node.getSpan() + " outside the sentence's " + starts.length + + " tokens"); + } + int low = 0; + int high = starts.length - 1; + while (low < high) { + final int mid = (low + high + 1) >>> 1; + if (starts[mid] <= offset) { + low = mid; + } else { + high = mid - 1; + } + } + return low; + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(PHRASES); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java new file mode 100644 index 0000000000..feb34b6aa4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.postag; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link POSTagger} to the document pipeline: reads {@link Layers#SENTENCES} + * and {@link Layers#TOKENS} and provides {@link Layers#POS_TAGS}, one tag annotation per + * token on the token's span. + * + *

Each sentence is tagged separately, the way the tagger contract expects its input, + * so tagging decisions never cross a sentence boundary. Token spans already refer to the + * original document text, so only the token sequence handed to the tagger is sliced per + * sentence; the produced tag layer stays aligned with {@link Layers#TOKENS} by + * position.

+ * + * @since 3.0.0 + */ +public final class POSTaggerAnnotator implements DocumentAnnotator { + + private final POSTagger tagger; + + /** + * Initializes the adapter. + * + * @param tagger The tagger to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code tagger} is {@code null}. + */ + public POSTaggerAnnotator(POSTagger tagger) { + if (tagger == null) { + throw new IllegalArgumentException("tagger must not be null"); + } + this.tagger = tagger; + } + + /** + * Tags the document sentence by sentence and adds the {@link Layers#POS_TAGS} layer. + * + *

For every sentence, the tokens whose spans lie inside the sentence span are + * tagged as one sequence, and each tag is emitted on its token's span. The required + * layers must be present, but they may be empty: a document without sentences or + * tokens yields a present-but-empty tag layer, and a sentence containing no tokens + * contributes nothing.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link Layers#POS_TAGS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * sentence layer or the token layer is absent, a token lies outside every + * sentence, or the tagger does not return one tag per token of a sentence. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> tagAnnotations = new ArrayList<>(tokens.size()); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final String[] tags = tagger.tag(words); + if (tags.length != words.length) { + throw new IllegalArgumentException( + "tagger returned " + tags.length + " tags for " + words.length + " tokens"); + } + for (int i = 0; i < words.length; i++) { + tagAnnotations.add(new Annotation<>(tokens.get(first + i).span(), tags[i])); + } + }); + return document.with(Layers.POS_TAGS, tagAnnotations); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(Layers.POS_TAGS); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java new file mode 100644 index 0000000000..0586daedc3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.sentdetect; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link SentenceDetector} to the document pipeline: provides + * {@link Layers#SENTENCES} from the document text. + * + *

The wrapped detector stays the primary API for single-task use; this adapter calls + * it like any other caller would.

+ * + * @since 3.0.0 + */ +public final class SentenceDetectorAnnotator implements DocumentAnnotator { + + private final SentenceDetector detector; + + /** + * Initializes the adapter. + * + * @param detector The sentence detector to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code detector} is {@code null}. + */ + public SentenceDetectorAnnotator(SentenceDetector detector) { + if (detector == null) { + throw new IllegalArgumentException("detector must not be null"); + } + this.detector = detector; + } + + /** + * Detects sentences over the document text and adds the {@link Layers#SENTENCES} + * layer, each sentence annotated with its covered text on its span. + * + * @param document The document to annotate. Must not be {@code null}. + * @return A new {@link Document} with the {@link Layers#SENTENCES} layer added. + * Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or + * already carries the {@link Layers#SENTENCES} layer. + */ + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final CharSequence text = document.text(); + final List> sentences = new ArrayList<>(); + for (final Span span : detector.sentPosDetect(text)) { + sentences.add(new Annotation<>(span, span.getCoveredText(text).toString())); + } + return document.with(Layers.SENTENCES, sentences); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(Layers.SENTENCES); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java new file mode 100644 index 0000000000..900fe188a6 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link Stemmer} to the document pipeline: stems the token layer and provides + * {@link #STEMS}, one annotation per token on the token's span. + * + *

Stemming operates on the token surface alone, so this annotator requires only the + * token layer; no part-of-speech tags are involved.

+ * + * @since 3.0.0 + */ +public final class StemmerAnnotator implements DocumentAnnotator { + + /** + * The stem layer. It is aligned with the token layer by position, and each annotation + * carries the stem of its token on that token's span. + */ + public static final LayerKey STEMS = Layers.key("stems", String.class); + + private final Stemmer stemmer; + + /** + * Initializes the adapter. + * + * @param stemmer The stemmer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code stemmer} is {@code null}. + */ + public StemmerAnnotator(Stemmer stemmer) { + if (stemmer == null) { + throw new IllegalArgumentException("stemmer must not be null"); + } + this.stemmer = stemmer; + } + + /** + * Stems the token layer and adds the {@link #STEMS} layer. + * + *

The token layer must be present, but it may be empty: a document without tokens + * yields a present-but-empty stem layer.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#TOKENS} layer. + * @return A new {@link Document} with the {@link #STEMS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or the + * token layer is absent. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.TOKENS); + final List> tokens = document.get(Layers.TOKENS); + final List> layer = new ArrayList<>(tokens.size()); + for (final Annotation token : tokens) { + layer.add(new Annotation<>(token.span(), stemmer.stem(token.value()).toString())); + } + return document.with(STEMS, layer); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(STEMS); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java new file mode 100644 index 0000000000..7760b565d3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link Tokenizer} to the document pipeline: provides {@link Layers#TOKENS}. + * + *

When {@link Layers#SENTENCES} is present, each sentence is tokenized separately and + * the token spans are shifted back to document coordinates; a present-but-empty sentence + * layer therefore yields a present-but-empty token layer. Only when the sentence layer + * is absent is the whole text tokenized at once. Either way, every token span refers to + * the original document text.

+ * + * @since 3.0.0 + */ +public final class TokenizerAnnotator implements DocumentAnnotator { + + private final Tokenizer tokenizer; + + /** + * Initializes the adapter. + * + * @param tokenizer The tokenizer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokenizer} is {@code null}. + */ + public TokenizerAnnotator(Tokenizer tokenizer) { + if (tokenizer == null) { + throw new IllegalArgumentException("tokenizer must not be null"); + } + this.tokenizer = tokenizer; + } + + /** + * Tokenizes the document and adds the {@link Layers#TOKENS} layer, sentence by + * sentence when a sentence layer is present and over the whole text otherwise. + * + * @param document The document to annotate. Must not be {@code null}. + * @return A new {@link Document} with the {@link Layers#TOKENS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or + * already carries the {@link Layers#TOKENS} layer. + */ + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final String text = document.text().toString(); + final List> tokens = new ArrayList<>(); + if (!document.layers().contains(Layers.SENTENCES)) { + addTokens(tokens, text, 0); + } else { + for (final Annotation sentence : document.get(Layers.SENTENCES)) { + final Span span = sentence.span(); + addTokens(tokens, text.substring(span.getStart(), span.getEnd()), span.getStart()); + } + } + return document.with(Layers.TOKENS, tokens); + } + + /** + * Tokenizes one stretch of text and appends its tokens, shifted back into document + * coordinates. + * + * @param tokens The layer under construction. + * @param text The stretch to tokenize. + * @param offset The stretch's start offset in the document text. + */ + private void addTokens(List> tokens, String text, int offset) { + for (final Span span : tokenizer.tokenizePos(text)) { + final Span shifted = new Span(span.getStart() + offset, span.getEnd() + offset); + tokens.add(new Annotation<>(shifted, span.getCoveredText(text).toString())); + } + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(Layers.TOKENS); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java new file mode 100644 index 0000000000..3214c13897 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.chunker; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +public class ChunkerAnnotatorTest { + + /** + * A chunker that records every token and tag sequence it receives and answers with + * one {@code NP} chunk per run of {@code N}-initial tags, so slicing and span mapping + * are observable. Tests override {@link #chunkAsSpans(String[], String[])} where a + * deviant answer is the fixture. + */ + private static class RecordingChunker implements Chunker { + + private final List> tokenCalls = new ArrayList<>(); + private final List> tagCalls = new ArrayList<>(); + + @Override + public String[] chunk(String[] toks, String[] tags) { + throw new UnsupportedOperationException("the adapter only calls chunkAsSpans"); + } + + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + tokenCalls.add(List.of(toks)); + tagCalls.add(List.of(tags)); + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= tags.length; i++) { + final boolean noun = i < tags.length && tags[i].startsWith("N"); + if (noun && start < 0) { + start = i; + } else if (!noun && start >= 0) { + spans.add(new Span(start, i, "NP")); + start = -1; + } + } + return spans.toArray(new Span[0]); + } + + @Override + public Sequence[] topKSequences(String[] sentence, String[] tags) { + throw new UnsupportedOperationException("the adapter only calls chunkAsSpans"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, String[] tags, + double minSequenceScore) { + throw new UnsupportedOperationException("the adapter only calls chunkAsSpans"); + } + } + + private static List> tokens(String text, String... forms) { + final List> annotations = new ArrayList<>(forms.length); + int cursor = 0; + for (final String form : forms) { + final int start = text.indexOf(form, cursor); + annotations.add(new Annotation<>(new Span(start, start + form.length()), form)); + cursor = start + form.length(); + } + return annotations; + } + + private static List> values(List> tokens, + String... tags) { + final List> annotations = new ArrayList<>(tags.length); + for (int i = 0; i < tags.length; i++) { + annotations.add(new Annotation<>(tokens.get(i).span(), tags[i])); + } + return annotations; + } + + /** Two sentences whose noun runs straddle neither sentence boundary. */ + private static Document twoSentences() { + final String text = "Mary Jones leads Acme. She joined Acme Corp."; + final List> toks = tokens(text, + "Mary", "Jones", "leads", "Acme", ".", "She", "joined", "Acme", "Corp", "."); + return Document.of(text) + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 22), "s"), + new Annotation<>(new Span(23, 44), "s"))) + .with(Layers.TOKENS, toks) + .with(Layers.POS_TAGS, values(toks, + "NNP", "NNP", "VBZ", "NNP", ".", "PRP", "VBD", "NNP", "NNP", ".")); + } + + @Test + void testChunksEachSentenceOntoTokenSpans() { + final RecordingChunker chunker = new RecordingChunker(); + final Document document = new ChunkerAnnotator(chunker).annotate(twoSentences()); + + Assertions.assertEquals(List.of( + List.of("Mary", "Jones", "leads", "Acme", "."), + List.of("She", "joined", "Acme", "Corp", ".")), chunker.tokenCalls); + Assertions.assertEquals(List.of( + List.of("NNP", "NNP", "VBZ", "NNP", "."), + List.of("PRP", "VBD", "NNP", "NNP", ".")), chunker.tagCalls); + final List> chunks = document.get(ChunkerAnnotator.CHUNKS); + Assertions.assertEquals(List.of( + new Annotation<>(new Span(0, 10), "NP"), + new Annotation<>(new Span(17, 21), "NP"), + new Annotation<>(new Span(34, 43), "NP")), chunks); + Assertions.assertEquals("Acme Corp", document.text().subSequence(34, 43).toString()); + } + + @Test + void testEmptyLayersYieldEmptyChunkLayer() { + final Document document = new ChunkerAnnotator(new RecordingChunker()).annotate( + Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of())); + Assertions.assertTrue(document.layers().contains(ChunkerAnnotator.CHUNKS)); + Assertions.assertTrue(document.get(ChunkerAnnotator.CHUNKS).isEmpty()); + } + + @Test + void testLayerContract() { + final ChunkerAnnotator annotator = new ChunkerAnnotator(new RecordingChunker()); + Assertions.assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + annotator.requires()); + Assertions.assertEquals(Set.of(ChunkerAnnotator.CHUNKS), annotator.provides()); + Assertions.assertEquals("opennlp:chunks", ChunkerAnnotator.CHUNKS.id()); + Assertions.assertEquals("ChunkerAnnotator", annotator.toString()); + } + + @Test + void testRejectsNullChunkerAndMissingLayers() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new ChunkerAnnotator(null)); + final ChunkerAnnotator annotator = new ChunkerAnnotator(new RecordingChunker()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(null)); + final Document untagged = Document.of("Mary.") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "s"))) + .with(Layers.TOKENS, tokens("Mary.", "Mary", ".")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(untagged)); + } + + @Test + void testRejectsMisalignedTagLayer() { + final String text = "Mary."; + final List> toks = tokens(text, "Mary", "."); + final Document document = Document.of(text) + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "s"))) + .with(Layers.TOKENS, toks) + .with(Layers.POS_TAGS, values(toks, "NNP")); + final ChunkerAnnotator annotator = new ChunkerAnnotator(new RecordingChunker()); + final IllegalArgumentException rejection = Assertions.assertThrows( + IllegalArgumentException.class, () -> annotator.annotate(document)); + Assertions.assertTrue(rejection.getMessage().contains("aligned"), + rejection.getMessage()); + } + + @Test + void testRejectsChunksOutsideSentenceOrWithoutType() { + final ChunkerAnnotator outside = new ChunkerAnnotator(new RecordingChunker() { + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + return new Span[] {new Span(0, toks.length + 1, "NP")}; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> outside.annotate(twoSentences())); + final ChunkerAnnotator empty = new ChunkerAnnotator(new RecordingChunker() { + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + return new Span[] {new Span(1, 1, "NP")}; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> empty.annotate(twoSentences())); + final ChunkerAnnotator untyped = new ChunkerAnnotator(new RecordingChunker() { + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + return new Span[] {new Span(0, 1)}; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> untyped.annotate(twoSentences())); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java new file mode 100644 index 0000000000..4ded778794 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link ArcStandardOracle} derivations replay to the gold graph through + * {@link ArcStandardState}, and that non-projective input is rejected. + */ +public class ArcStandardOracleTest { + + /** + * Derives the oracle transitions for {@code gold} and replays them on a fresh state. + * + * @param gold The gold graph to derive from. + * @return The graph the replayed derivation builds. Never {@code null}. + */ + private static DependencyGraph replay(DependencyGraph gold) { + final List transitions = ArcStandardOracle.transitions(gold); + // every token is shifted once and attached once + assertEquals(2 * gold.size(), transitions.size()); + final ArcStandardState state = new ArcStandardState(gold.size()); + for (final Transition transition : transitions) { + assertTrue(state.canApply(transition)); + state.apply(transition); + } + return state.toGraph(); + } + + @Test + void testRoundTripSimpleSentence() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripSingleToken() { + final DependencyGraph gold = DependencyGraph.of(new int[] {-1}, new String[] {"root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripRightBranching() { + // "eat fresh fish now": root with a right dependent that has its own left dependent + final DependencyGraph gold = DependencyGraph.of(new int[] {-1, 2, 0, 0}, + new String[] {"root", "amod", "obj", "advmod"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripDeepChain() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"a", "b", "c", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testNonProjectiveThrows() { + // arcs (2,0) and (3,1) cross, so there is no arc-standard derivation + final DependencyGraph nonProjective = DependencyGraph.of(new int[] {2, 3, -1, 2}, + new String[] {"a", "b", "root", "c"}); + assertThrows(IllegalArgumentException.class, + () -> ArcStandardOracle.transitions(nonProjective)); + } + + @Test + void testNullGraphThrows() { + assertThrows(IllegalArgumentException.class, () -> ArcStandardOracle.transitions(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java new file mode 100644 index 0000000000..9cc6cc0b47 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the configuration mechanics of {@link ArcStandardState}: the start + * configuration, transition applicability at the boundaries, the bookkeeping of attached + * dependents, copy independence, and the fail-loud behavior of every accessor. + */ +public class ArcStandardStateTest { + + @Test + void testInitialConfiguration() { + final ArcStandardState state = new ArcStandardState(3); + assertEquals(ArcStandardState.ROOT, state.stack(0)); + assertEquals(ArcStandardState.NONE, state.stack(1)); + assertEquals(0, state.buffer(0)); + assertEquals(1, state.buffer(1)); + assertEquals(2, state.buffer(2)); + assertEquals(ArcStandardState.NONE, state.buffer(3)); + assertEquals(1, state.stackSize()); + assertEquals(3, state.bufferSize()); + assertFalse(state.isTerminal()); + } + + @Test + void testSingleTokenDerivationIsForced() { + // With one token the system permits exactly one derivation: shift the token, then + // attach it to the artificial root with a right arc. + final ArcStandardState state = new ArcStandardState(1); + assertTrue(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertFalse(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.SHIFT); + assertFalse(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertTrue(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + state.toGraph()); + } + + @Test + void testDependentBookkeepingDuringADerivation() { + // Derives "the dog barks" (the<-dog via det, dog<-barks via nsubj, barks<-root) and + // checks the partial-structure accessors after every attachment. + final ArcStandardState state = new ArcStandardState(3); + state.apply(Transition.SHIFT); + state.apply(Transition.SHIFT); + assertEquals(0, state.assignedDependents(1)); + assertNull(state.assignedRelation(0)); + + state.apply(Transition.leftArc("det")); + assertEquals(1, state.assignedDependents(1)); + assertEquals(0, state.leftmostDependent(1)); + assertEquals(0, state.rightmostDependent(1)); + assertEquals("det", state.assignedRelation(0)); + + state.apply(Transition.SHIFT); + state.apply(Transition.leftArc("nsubj")); + assertEquals(1, state.leftmostDependent(2)); + assertEquals("nsubj", state.assignedRelation(1)); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), state.toGraph()); + } + + @Test + void testInapplicableTransitionFailsLoud() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.leftArc("det"))); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.rightArc("root"))); + assertThrows(IllegalArgumentException.class, () -> state.apply(null)); + assertThrows(IllegalArgumentException.class, () -> state.canApply(null)); + } + + @Test + void testToGraphBeforeTerminalFailsLoud() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalStateException.class, state::toGraph); + state.apply(Transition.SHIFT); + assertThrows(IllegalStateException.class, state::toGraph); + } + + @Test + void testCopyIsIndependentOfTheOriginal() { + final ArcStandardState original = new ArcStandardState(2); + original.apply(Transition.SHIFT); + final ArcStandardState copy = original.copy(); + + copy.apply(Transition.SHIFT); + copy.apply(Transition.leftArc("nsubj")); + // The copy advanced by two transitions while the original still has one token + // buffered and one on the stack. + assertEquals(2, original.stackSize()); + assertEquals(1, original.bufferSize()); + assertEquals(0, original.assignedDependents(1)); + assertEquals(1, copy.assignedDependents(1)); + } + + @Test + void testAccessorValidation() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, () -> state.stack(-1)); + assertThrows(IllegalArgumentException.class, () -> state.buffer(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(2)); + assertThrows(IllegalArgumentException.class, () -> state.leftmostDependent(2)); + assertThrows(IllegalArgumentException.class, () -> state.rightmostDependent(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedRelation(2)); + } + + @Test + void testTokenCountMustBePositive() { + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(-1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java new file mode 100644 index 0000000000..fea7b8137c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pins down the boundary behavior of {@link DependencyAnnotator}: the exact exception and + * message for absent, empty, and misaligned input layers, the immutability rule that a + * second annotation pass is rejected, and the exact {@code requires()} and + * {@code provides()} declarations the pipeline validation relies on. + */ +public class DependencyAnnotatorEdgeCaseTest { + + /** The rejection message for a token that no sentence encloses. */ + private static final String STRAY_TOKEN = "token at [3..5) lies outside every sentence"; + + /** The rejection message for a tag layer that does not have one tag per token. */ + private static final String MISALIGNED = + "document needs aligned opennlp:tokens and opennlp:pos layers"; + + /** + * A parser stub that returns a fixed two-token graph regardless of its input, so the + * assertions in this class exercise only the annotator's own layer handling. + */ + private static final DependencyParser FIXED = (tokens, tags) -> + DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, + new String[] {"root", "obj"}); + + /** + * A parser stub that accepts only one-token sentences and returns their single root + * arc, so a sentence slice of any other length fails the test loudly. + */ + private static final DependencyParser ONE_TOKEN_ROOT = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + + /** + * A parser stub that returns a flat tree of the requested size, for assertions that + * must get past the annotator's graph-size validation with sentences of any length. + */ + private static final DependencyParser SIZE_MATCHING = (tokens, tags) -> { + final int[] heads = new int[tokens.length]; + final String[] relations = new String[tokens.length]; + heads[0] = DependencyArc.ROOT_HEAD; + relations[0] = "root"; + for (int i = 1; i < heads.length; i++) { + heads[i] = 0; + relations[i] = "dep"; + } + return DependencyGraph.of(heads, relations); + }; + + /** + * Builds a document over the text {@code "ab cd"} carrying aligned two-entry token and + * tag layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * + * @return A document ready for dependency annotation. Never {@code null}. + */ + private static Document twoTokens() { + return Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "ab cd"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + } + + /** + * Supplies one document per required layer, each missing exactly that layer, together + * with the key the rejection message must name. + * + * @return The documents and the expected layer key. Never {@code null}. + */ + private static Stream documentsMissingOneLayer() { + final List> sentence = + List.of(new Annotation<>(new Span(0, 5), "ab cd")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd")); + final List> tags = List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN")); + return Stream.of( + Arguments.of(Document.of("ab cd") + .with(Layers.TOKENS, tokens).with(Layers.POS_TAGS, tags), Layers.SENTENCES), + Arguments.of(Document.of("ab cd") + .with(Layers.SENTENCES, sentence).with(Layers.POS_TAGS, tags), Layers.TOKENS), + Arguments.of(Document.of("ab cd") + .with(Layers.SENTENCES, sentence).with(Layers.TOKENS, tokens), Layers.POS_TAGS)); + } + + @ParameterizedTest + @MethodSource("documentsMissingOneLayer") + void testAbsentRequiredLayerIsNamed(Document document, LayerKey missing) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(document)); + assertEquals("document lacks the required layer " + missing, e.getMessage()); + } + + @Test + void testNullDocumentIsRejected() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(null)); + assertEquals("document must not be null", e.getMessage()); + } + + /** + * Verifies the empty-versus-absent distinction of the annotator contract: present but + * empty required layers are valid input and yield a present-but-empty arc layer, so a + * pipeline does not fail on a document without content. + */ + @Test + void testEmptyRequiredLayersYieldAnEmptyArcLayer() { + final Document empty = Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of()); + final Document annotated = new DependencyAnnotator(FIXED).annotate(empty); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, + DependencyAnnotator.DEPENDENCIES), annotated.layers()); + assertEquals(List.of(), annotated.get(DependencyAnnotator.DEPENDENCIES)); + } + + @Test + void testMisalignedTagLayerIsRejected() { + // two tokens but only one tag: the layers are present yet not aligned by position + final Document misaligned = Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "ab cd"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(misaligned)); + assertEquals(MISALIGNED, e.getMessage()); + } + + @Test + void testAnnotatingTwiceIsRejected() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + final Document once = annotator.annotate(twoTokens()); + assertEquals(2, once.get(DependencyAnnotator.DEPENDENCIES).size()); + + // documents are immutable and layers are add-once: a second pass must not overwrite + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(once)); + assertEquals("layer is already present: opennlp:dependencies", e.getMessage()); + } + + @Test + void testRequiresAndProvidesDeclarationsAreExact() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + annotator.requires()); + assertEquals(Set.of(DependencyAnnotator.DEPENDENCIES), annotator.provides()); + } + + /** + * Verifies the per-sentence contract: two one-token sentences are parsed as two + * separate calls, each yielding its own root arc, and the dependents come back as + * document-wide token indices. + */ + @Test + void testEachSentenceGetsItsOwnTree() { + final Document document = Document.of("ab. cd.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "ab."), + new Annotation<>(new Span(4, 7), "cd."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(4, 6), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(4, 6), "VB"))); + + final List> arcs = + new DependencyAnnotator(ONE_TOKEN_ROOT).annotate(document) + .get(DependencyAnnotator.DEPENDENCIES); + assertEquals(2, arcs.size()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 0, "root"), + arcs.get(0).value()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), + arcs.get(1).value()); + assertEquals(new Span(0, 2), arcs.get(0).span()); + assertEquals(new Span(4, 6), arcs.get(1).span()); + } + + /** + * Verifies a token that no sentence encloses is reported instead of being parsed + * outside of any sentence. + */ + @Test + void testTokenOutsideEverySentenceIsRejected() { + final Document strayToken = Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 2), "ab"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(strayToken)); + assertEquals(STRAY_TOKEN, stray.getMessage()); + } + + @Test + void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { + // requires() feeds the analyzer's validation: no tokenizer or tagger, no pipeline + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new DependencyAnnotator(FIXED)); + assertThrows(IllegalArgumentException.class, builder::build); + } + + /** + * Verifies the javadoc-promised behavior for a sentence containing no tokens: it + * contributes no arcs and no parser call, and the token indices of the sentence + * after it still shift by the correct first-token position rather than by a count + * that includes the empty sentence. + */ + @Test + void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { + final Document document = Document.of("ab. ??? cd.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "ab."), + new Annotation<>(new Span(4, 7), "???"), + new Annotation<>(new Span(8, 11), "cd."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(8, 10), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(8, 10), "VB"))); + + final List> arcs = + new DependencyAnnotator(ONE_TOKEN_ROOT).annotate(document) + .get(DependencyAnnotator.DEPENDENCIES); + assertEquals(2, arcs.size()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), + arcs.get(1).value()); + assertEquals(new Span(8, 10), arcs.get(1).span()); + } + + /** + * Verifies the text-order walk on a token straddling two sentence spans: the token + * belongs to neither sentence under the enclosure rule, the scan sticks at it, and + * the annotator reports it as lying outside every sentence instead of silently + * attaching it to one of its neighbors. + */ + @Test + void testTokenStraddlingTwoSentencesIsRejected() { + final Document document = Document.of("ab cd ef") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 4), "ab c"), + new Annotation<>(new Span(4, 8), "d ef"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); + assertEquals(STRAY_TOKEN, e.getMessage()); + } + + /** + * Verifies the stuck-scan path: a gap token between sentences stops the walk, and + * the token-bearing sentence after the gap does not pull the scan forward past the + * stray token, which is still reported rather than skipped. + */ + @Test + void testGapTokenBeforeATokenBearingSentenceIsStillRejected() { + final Document document = Document.of("ab cd ef.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(6, 9), "ef."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); + assertEquals(STRAY_TOKEN, e.getMessage()); + } + + /** + * Verifies that a parser returning a wrong-size graph is rejected loudly instead of + * silently misaligning the dependency layer with the token layer: the fixed + * two-token stub meets a three-token sentence and the annotator names both counts. + */ + @Test + void testWrongSizeGraphFailsLoud() { + final Document document = Document.of("ab cd ef") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 8), "ab cd ef"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(document)); + assertEquals("parser returned a graph over 2 tokens for a sentence of 3", + e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java new file mode 100644 index 0000000000..46d97fafd4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.postag.POSTagger; +import opennlp.tools.postag.POSTaggerAnnotator; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.sentdetect.SentenceDetectorAnnotator; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.TokenizerAnnotator; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Demonstrates {@link DependencyAnnotator} at the end of a complete {@link DocumentAnalyzer} + * pipeline: raw text goes in, and the dependency layer comes out anchored on the original + * text. The upstream steps are deliberately simple inline implementations of the task + * interfaces, and the parser is a {@link DependencyParserME} trained here on a tiny corpus + * it can memorize, so every expected head, relation, and span is exact and reproducible. + * + *

The central property under test is coordinate anchoring for multi-sentence input: the + * annotator parses each sentence separately and shifts the sentence-local arcs by the + * sentence's first token position, so every head and dependent is an index into the + * document-wide token layer and each arc's annotation sits on its dependent token's span in + * document coordinates. For the second sentence this only works when the shift is applied + * consistently, which the assertions verify by reading the covered text of the arcs' spans + * back out of the original document.

+ */ +public class DependencyAnnotatorPipelineTest { + + /** + * The two-sentence input text; sentence one covers offsets 0..14 and sentence two covers + * offsets 15..29 of the original document. + */ + private static final String TEXT = "the dog barks. she eats fish."; + + /** + * Maps every token of the test corpus to its part-of-speech tag, standing in for a + * trained tagger. + */ + private static final Map LEXICON = Map.of( + "the", "DT", "dog", "NN", "barks", "VBZ", + "she", "PRP", "eats", "VBZ", "fish", "NN"); + + /** + * A sentence detector stub that closes a sentence after every period and skips the one + * following blank, producing sentence spans in document coordinates. + */ + private static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the pipeline only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A tokenizer stub that treats blanks and periods as token boundaries, so word tokens + * come out without trailing punctuation. + */ + private static final Tokenizer WORD_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the pipeline only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = + i == s.length() || s.charAt(i) == ' ' || s.charAt(i) == '.'; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A dictionary tagger over {@link #LEXICON} that fails loud on any token the test corpus + * does not define, so a tokenization mistake cannot silently degrade the parse. + */ + private static final POSTagger LEXICON_TAGGER = new POSTagger() { + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + final String tag = LEXICON.get(sentence[i]); + if (tag == null) { + throw new IllegalArgumentException("token is not in the test lexicon: " + sentence[i]); + } + tags[i] = tag; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the pipeline only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the pipeline only calls tag"); + } + }; + + private static DependencyParserME parser; + + /** + * Builds the training corpus: the two sentences of the example document with their + * gold trees, plus a one-token sentence, each repeated often enough for the model to + * memorize them. Every sample is one sentence, the unit the annotator hands to the + * parser, so each sentence carries its own root. + * + * @return The training samples. Never {@code null} or empty. + */ + private static List corpus() { + final List distinct = List.of( + new DependencySample( + new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}, + DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"})), + new DependencySample( + new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"}, + DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"})), + new DependencySample(new String[] {"barks"}, new String[] {"VBZ"}, + DependencyGraph.of(new int[] {-1}, new String[] {"root"}))); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + /** + * Trains the shared parser once for all tests. The trainer is deterministic for a fixed + * corpus and fixed parameters, so the assertions below hold on every run. + * + * @throws IOException Thrown if training fails, which fails the test class. + */ + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + parser = new DependencyParserME(DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters)); + } + + /** + * Assembles the complete pipeline: sentence splitting, tokenization, tagging, and + * dependency parsing with the trained model. + * + * @return A {@link DocumentAnalyzer} ready to analyze raw text. Never {@code null}. + */ + private static DocumentAnalyzer pipeline() { + return DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(WORD_TOKENIZER)) + .add(new POSTaggerAnnotator(LEXICON_TAGGER)) + .add(new DependencyAnnotator(parser)) + .build(); + } + + @Test + void testTwoSentenceTextYieldsOneExactArcPerTokenInDocumentCoordinates() { + final Document document = pipeline().analyze(TEXT); + + // sanity of the upstream layers the dependency annotator consumed + assertEquals(2, document.get(Layers.SENTENCES).size()); + assertEquals(6, document.get(Layers.TOKENS).size()); + + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(6, arcs.size()); + + // the memorized gold trees, one root per sentence, every span and head index in + // document coordinates + final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, DependencyArc.ROOT_HEAD, 4}; + final String[] relations = {"det", "nsubj", "root", "nsubj", "root", "obj"}; + final Span[] spans = {new Span(0, 3), new Span(4, 7), new Span(8, 13), + new Span(15, 18), new Span(19, 23), new Span(24, 28)}; + for (int i = 0; i < arcs.size(); i++) { + final Annotation arc = arcs.get(i); + assertEquals(spans[i], arc.span(), "span of arc " + i); + assertEquals(heads[i], arc.value().head(), "head of arc " + i); + assertEquals(i, arc.value().dependent(), "dependent of arc " + i); + assertEquals(relations[i], arc.value().relation(), "relation of arc " + i); + } + } + + @Test + void testSecondSentenceArcsResolveToTheOriginalText() { + final Document document = pipeline().analyze(TEXT); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + + // "she" is token 3 of the document-wide token layer, not token 0 of its sentence + final Annotation she = arcs.get(3); + assertEquals(new Span(15, 18), she.span()); + assertEquals("she", she.span().getCoveredText(document.text()).toString()); + assertEquals("nsubj", she.value().relation()); + + // its head index is likewise document-wide: 4 points at "eats", never 1 at "dog" + assertEquals(4, she.value().head()); + final Annotation head = document.get(Layers.TOKENS).get(she.value().head()); + assertEquals("eats", head.value()); + assertEquals(new Span(19, 23), head.span()); + assertEquals("eats", head.span().getCoveredText(document.text()).toString()); + } + + @Test + void testSingleTokenSentenceParsesToARootArc() { + final Document document = pipeline().analyze("barks."); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(1, arcs.size()); + assertEquals(new Span(0, 5), arcs.get(0).span()); + assertEquals(DependencyArc.ROOT_HEAD, arcs.get(0).value().head()); + assertEquals(0, arcs.get(0).value().dependent()); + assertEquals("root", arcs.get(0).value().relation()); + } + + @Test + void testTextWithZeroSentencesYieldsAnEmptyDependencyLayer() { + // empty text yields present-but-empty sentence, token, and tag layers, which every + // annotator of the pipeline passes through under the empty-versus-absent + // distinction, so the dependency layer comes out present and empty + final Document document = pipeline().analyze(""); + assertEquals(List.of(), document.get(Layers.TOKENS)); + assertEquals(List.of(), document.get(DependencyAnnotator.DEPENDENCIES)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java new file mode 100644 index 0000000000..c5b215452b --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyAnnotator} as the container's first graph-shaped layer: arcs + * reference tokens by layer index, and resolving an arc through the token layer lands on + * the right span of the original text. + */ +public class DependencyAnnotatorTest { + + /** + * A parser stub that always returns the gold graph of {@code "the dog barks"}, so the + * assertions in this class depend only on the annotator's own layer handling and not on + * any trained model. + */ + private static final DependencyParser FIXED = (tokens, tags) -> + DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + + /** + * Builds a document over the text {@code "the dog barks"} carrying a one-sentence layer plus + * aligned token and tag layers, mirroring what the upstream sentence, tokenizer, and + * tagger annotators would produce. + * + * @return A document ready for dependency annotation. Never {@code null}. + */ + private static Document tokenized() { + return Document.of("the dog barks") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 13), "the dog barks"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(8, 13), "barks"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "DT"), + new Annotation<>(new Span(4, 7), "NN"), + new Annotation<>(new Span(8, 13), "VBZ"))); + } + + @Test + void testArcsResolveThroughTheTokenLayer() { + final Document document = new DependencyAnnotator(FIXED).annotate(tokenized()); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(3, arcs.size()); + + // the arc of "dog" is anchored on the dependent token's span in the original text + final Annotation dog = arcs.get(1); + assertEquals(new Span(4, 7), dog.span()); + assertEquals("nsubj", dog.value().relation()); + + // cross-layer reference: the arc stores its head as an index, and looking that index + // up in the token layer lands on the head token and its span in the original text + final List> tokens = document.get(Layers.TOKENS); + final Annotation head = tokens.get(dog.value().head()); + assertEquals("barks", head.value()); + assertEquals("barks", head.span().getCoveredText(document.text()).toString()); + } + + @Test + void testRootArcCarriesRootHead() { + final Document document = new DependencyAnnotator(FIXED).annotate(tokenized()); + final DependencyArc root = + document.get(DependencyAnnotator.DEPENDENCIES).get(2).value(); + assertEquals(DependencyArc.ROOT_HEAD, root.head()); + assertEquals("root", root.relation()); + } + + @Test + void testMissingLayersThrow() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no layers"))); + assertEquals("document lacks the required layer " + Layers.SENTENCES, e.getMessage()); + } + + @Test + void testNullParserThrows() { + assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java new file mode 100644 index 0000000000..4d46e04022 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the boundary behavior of both dependency parsers: empty and single-token input, + * non-projective sentences during training and parsing, and the file round trip of both + * model formats, which must reproduce the exact parses of the original models. + */ +public class DependencyParserEdgeCaseTest { + + private static DependencyModel maxentModel; + private static DependencyParserME maxentParser; + private static FeedforwardDependencyModel feedforwardModel; + private static FeedforwardDependencyParser feedforwardParser; + + /** + * Builds a four-token sample whose gold arcs (2,0) and (3,1) cross, so the tree is + * non-projective and has no arc-standard derivation. + * + * @return The non-projective sample. Never {@code null}. + */ + private static DependencySample nonProjectiveSample() { + return sample(new String[] {"the", "dog", "barks", "today"}, + new String[] {"DT", "NN", "VBZ", "RB"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "nsubj", "root", "advmod"}); + } + + /** + * Trains one classical and one neural model on the shared corpus. The feedforward + * settings disable dropout and fix the seed, so the tiny network memorizes the corpus + * deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParsers() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + maxentModel = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters); + maxentParser = new DependencyParserME(maxentModel); + + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + feedforwardModel = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + feedforwardParser = new FeedforwardDependencyParser(feedforwardModel); + } + + @Test + void testEmptySentenceIsRejectedByBothParsers() { + assertThrows(IllegalArgumentException.class, + () -> maxentParser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> feedforwardParser.parse(new String[0], new String[0])); + // The transition system itself has no configuration for zero tokens either. + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + } + + @Test + void testSingleTokenSentenceAttachesToTheRoot() { + // A single token permits only the derivation shift then right-arc, so the head is + // forced to the artificial root and the model only chooses the relation label. + final DependencyGraph maxentParse = + maxentParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), maxentParse); + + final DependencyGraph feedforwardParse = + feedforwardParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + feedforwardParse); + } + + @Test + void testNonProjectiveSamplesAreSkippedDuringTraining() throws IOException { + // One non-projective sample joins the corpus; it cannot yield events, so training + // proceeds on the remaining samples and still memorizes the projective sentences. + final List mixed = new ArrayList<>(corpus()); + mixed.add(nonProjectiveSample()); + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + final DependencyModel model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(mixed), parameters); + final DependencyParserME parser = new DependencyParserME(model); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + parser.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"})); + } + + @Test + void testNonProjectiveGoldDecodesToAProjectiveTree() { + // The parser can only emit arc-standard derivations, so for a sentence whose gold + // tree is non-projective the prediction is necessarily a different, projective tree. + final DependencySample gold = nonProjectiveSample(); + final DependencyGraph parsed = maxentParser.parse(gold.getTokens(), gold.getTags()); + assertNotEquals(gold.getGraph(), parsed); + assertEquals(0, crossingArcCount(parsed)); + // The unseen final token becomes the root and the verb attaches under it; the + // familiar determiner and subject arcs survive from the training evidence. + assertEquals(DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"det", "nsubj", "nsubj", "root"}), parsed); + } + + @Test + void testFeedforwardTrainingFailsLoudWithoutProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(8, 8, 1, 32, 0.05, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings)); + assertEquals("no trainable examples in the samples", e.getMessage()); + } + + @Test + void testRefinementFailsLoudWithoutProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(feedforwardModel, + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings, 2)); + assertEquals("no trainable samples for refinement", e.getMessage()); + } + + @Test + void testMaxentModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse.bin"); + maxentModel.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + for (final DependencySample sample : corpus()) { + assertEquals(maxentParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags())); + } + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + reloaded.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + @Test + void testFeedforwardModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse-ff.bin"); + try (OutputStream out = Files.newOutputStream(file)) { + feedforwardModel.serialize(out); + } + final FeedforwardDependencyParser reloaded = + new FeedforwardDependencyParser(FeedforwardDependencyModel.load(file)); + for (final DependencySample sample : corpus()) { + assertEquals(feedforwardParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags())); + } + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"})); + } + + /** + * Counts the pairs of crossing arcs in a graph, treating the root arc as spanning + * from a virtual position left of the sentence to its dependent. A projective tree + * has zero crossing pairs. + * + * @param graph The graph to inspect. Must not be {@code null}. + * @return The number of crossing arc pairs. + * @throws IllegalArgumentException Thrown if {@code graph} is {@code null}. + */ + private static int crossingArcCount(DependencyGraph graph) { + if (graph == null) { + throw new IllegalArgumentException("graph must not be null"); + } + int crossings = 0; + for (int i = 0; i < graph.size(); i++) { + for (int j = i + 1; j < graph.size(); j++) { + final int iLow = Math.min(i, graph.headOf(i)); + final int iHigh = Math.max(i, graph.headOf(i)); + final int jLow = Math.min(j, graph.headOf(j)); + final int jHigh = Math.max(j, graph.headOf(j)); + if ((iLow < jLow && jLow < iHigh && iHigh < jHigh) + || (jLow < iLow && iLow < jHigh && jHigh < iHigh)) { + crossings++; + } + } + } + return crossings; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java new file mode 100644 index 0000000000..aa839c5fbf --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyParserME} end to end: training on a tiny corpus must let the + * greedy parser reproduce the training sentences, which proves the oracle, event stream, + * feature generation, and decode loop agree with each other. + */ +public class DependencyParserMETest { + + private static DependencyModel model; + private static DependencyParserME parser; + + /** + * Trains the shared model once for all tests; the zero cutoff keeps every feature of + * the tiny corpus. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters); + parser = new DependencyParserME(model); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testParseAlwaysYieldsASingleRootedTree() { + // an unseen sentence must still decode to a valid graph, whatever its quality + final DependencyGraph parsed = parser.parse(new String[] {"cats", "sleep"}, + new String[] {"NNS", "VBP"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(320, evaluator.getWordCount()); + } + + @Test + void testParseValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } + + @Test + void testConstructorRejectsNullModel() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((DependencyModel) null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((MaxentModel) null)); + } + + @Test + void testTrainValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train("eng", null, TrainingParameters.defaultParams())); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(null, + ObjectStreamUtils.createObjectStream(corpus()), + TrainingParameters.defaultParams())); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + final DependencyModel reloaded = new DependencyModel( + new ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new DependencyParserME(reloaded) + .parse(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testModelRejectsNullParserModel() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel("eng", null, null)); + } + + @Test + void testModelWithForeignOutcomesIsRejectedAtConstruction() { + // The outcome inventory is decoded once up front, so a model trained for another + // task is rejected when the parser is built rather than mid-sentence. + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME(new OutcomeOnlyModel("NN", "VB"))); + } + + /** + * A {@link MaxentModel} that only knows its outcome inventory, enough to build a + * parser from; any other use fails. + */ + private record OutcomeOnlyModel(String... outcomes) implements MaxentModel { + + @Override + public String getOutcome(int i) { + return outcomes[i]; + } + + @Override + public int getNumOutcomes() { + return outcomes.length; + } + + @Override + public double[] eval(String[] context) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, double[] probs) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, float[] values) { + throw new UnsupportedOperationException(); + } + + @Override + public String getBestOutcome(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public String getAllOutcomes(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public int getIndex(String outcome) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java new file mode 100644 index 0000000000..051668a585 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * The gold samples shared by the dependency parser tests. + */ +final class DependencyTestSamples { + + /** How often the distinct sentences are repeated in {@link #corpus()}. */ + private static final int REPETITIONS = 40; + + private DependencyTestSamples() { + // This class only exposes static sample builders and is never instantiated. + } + + /** + * Builds one gold sample from its parallel arrays. + * + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @param heads The zero-based head per token, {@code -1} for the root. + * @param relations The relation label per token. + * @return The assembled sample. Never {@code null}. + */ + static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + /** + * Builds the projective training corpus: three tiny sentences, each repeated so both + * trainers see enough evidence to memorize them. + * + * @return The training samples. Never {@code null}. + */ + static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(REPETITIONS * distinct.size()); + for (int i = 0; i < REPETITIONS; i++) { + corpus.addAll(distinct); + } + return corpus; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java new file mode 100644 index 0000000000..6d7437587e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.StringUtil; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the pure-Java neural tier end to end: training on a tiny corpus must let the + * greedy feedforward parser reproduce the training sentences, and a model must survive + * the serialization round trip bit-for-bit in behavior. + */ +public class FeedforwardDependencyParserTest { + + private static FeedforwardDependencyModel model; + private static FeedforwardDependencyParser parser; + + /** + * Trains the shared model once for all tests, with dropout off and a fixed seed so + * the tiny network memorizes the corpus deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 120, 32, 0.05, 0.0, 0.0, 1, 17L); + model = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + parser = new FeedforwardDependencyParser(model); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + @Test + void testUnknownWordsStillYieldASingleRootedTree() { + final DependencyGraph parsed = parser.parse(new String[] {"unseen", "words"}, + new String[] {"JJ", "NNS"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testBeamOfOneMatchesGreedy() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 1); + for (final DependencySample sample : corpus()) { + assertEquals(parser.parse(sample.getTokens(), sample.getTags()), + beamed.parse(sample.getTokens(), sample.getTags())); + } + } + + @Test + void testBeamedParserReproducesTrainingSentences() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 4); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + beamed.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + @Test + void testBeamedParseIsDeterministicAndSingleRooted() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 8); + final String[] tokens = {"unseen", "words", "everywhere"}; + final String[] tags = {"JJ", "NNS", "RB"}; + final DependencyGraph first = beamed.parse(tokens, tags); + assertEquals(first, beamed.parse(tokens, tags)); + assertEquals(3, first.size()); + first.root(); + } + + @Test + void testRefinementKeepsToyPerformance() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final FeedforwardDependencyTrainer.Settings refineSettings = + new FeedforwardDependencyTrainer.Settings(16, 32, 2, 32, 0.01, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new FeedforwardDependencyParser(refined, 2)); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + @Test + void testRefineWithAnUnknownRelationFailsLoud() throws IOException { + // A refinement corpus may carry a relation label the original training set never + // used; the transition inventory is fixed at training time, so refinement cannot + // score it and must say which transition it does not know. + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final List unseenRelation = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "dislocated", "root"})); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(model, + ObjectStreamUtils.createObjectStream(unseenRelation), settings, 2)); + assertEquals("unknown transition in the refinement samples: LEFT_ARC:dislocated", + e.getMessage()); + } + + @Test + void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = local.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + final double[] before = local.score(features); + + final FeedforwardDependencyTrainer.Settings refineSettings = + new FeedforwardDependencyTrainer.Settings(16, 32, 2, 32, 0.01, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + // refinement produces a distinct model, so a model already shared between threads + // cannot change underneath them + assertNotSame(local, refined); + assertArrayEquals(before, local.score(features)); + // and the returned model really carries the refinement + assertFalse(Arrays.equals(before, refined.score(features))); + } + + /** + * Pins the one contextual rule layered over the per-code-point mapping: a Greek + * capital sigma preceded by a letter and not followed by one lowercases to the + * final form U+03C2, the way natural lowercase Greek spells it and the way every + * treebank-derived vocabulary key spells it, so an uppercase Greek word normalizes + * to a key the vocabulary can actually contain. The plain per-code-point mapping + * would produce the medial sigma there and miss the vocabulary. + */ + @Test + void testNormalizeAppliesTheFinalSigmaRule() { + // ODOS, road, all caps: the trailing sigma position lowers to U+03C2 + assertEquals("\u03BF\u03B4\u03BF\u03C2", + FeedforwardDependencyModel.normalize("\u039F\u0394\u039F\u03A3")); + // SOFIA: the word-initial sigma is not final and lowers to the medial U+03C3 + assertEquals("\u03C3\u03BF\u03C6\u03B9\u03B1", + FeedforwardDependencyModel.normalize("\u03A3\u039F\u03A6\u0399\u0391")); + // a lone capital sigma has no preceding letter, so the rule does not fire + assertEquals("\u03C3", FeedforwardDependencyModel.normalize("\u03A3")); + } + + /** + * Pins the allocation-free fast path: a word the mapping leaves unchanged, the + * overwhelming majority of parse-time input, is returned as the same instance + * rather than a fresh copy built on every lookup of the scoring loop. + */ + @Test + void testNormalizeReturnsTheSameInstanceForLowercaseWords() { + final String plain = "barks"; + assertSame(plain, FeedforwardDependencyModel.normalize(plain)); + // lowercase Greek with its native final sigma is already normalized + final String greek = "\u03BF\u03B4\u03BF\u03C2"; + assertSame(greek, FeedforwardDependencyModel.normalize(greek)); + } + + /** + * Pins the refinement contract for the sample kind the unknown-transition check + * never sees: a non-projective gold graph has no arc-standard derivation and is + * skipped before the transition inventory is consulted, so an unknown relation + * riding on it must not trigger the unknown-transition failure, and refinement + * proceeds on the remaining projective samples. + */ + @Test + void testNonProjectiveSampleWithUnknownRelationIsSkippedNotFatal() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + // heads {2, 3, -1, 2}: the arcs from 2 to 0 and from 3 to 1 cross, so the graph + // is non-projective, and "dislocated" is a relation the model was never trained on + final List mixed = List.of( + sample(new String[] {"a", "b", "c", "d"}, new String[] {"DT", "NN", "VBZ", "NN"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "dislocated", "root", "obj"}), + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"})); + + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + model, ObjectStreamUtils.createObjectStream(mixed), settings, 2); + assertNotSame(model, refined); + } + + /** + * Pins the serialized vocabulary order: entries are written in ascending id order, + * not in the iteration order of the underlying immutable maps, which the JDK salts + * per launch, so serializing the same model produces the same bytes on every run. + */ + @Test + void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + try (DataInputStream data = new DataInputStream( + new ByteArrayInputStream(out.toByteArray()))) { + data.readUTF(); + for (int vocab = 0; vocab < 3; vocab++) { + final int size = data.readInt(); + int previous = Integer.MIN_VALUE; + for (int entry = 0; entry < size; entry++) { + data.readUTF(); + final int id = data.readInt(); + assertTrue(id > previous, + "vocabulary " + vocab + " must be written in ascending id order"); + previous = id; + } + } + } + } + + /** + * Pins the scoring cache against the direct path: the parser built in setup turned + * the cache on for the shared model, so scoring the same configuration through a + * fresh uncached copy must agree to float rounding, and the winning transition must + * be identical. Repeated scoring exercises the cache-hit path as well as the + * first-sight fill. + */ + @Test + void testScoringCacheMatchesTheDirectPath() { + final FeedforwardDependencyModel uncached = model.copy(); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = model.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + + for (int round = 0; round < 3; round++) { + final double[] cached = model.score(features); + final double[] direct = uncached.score(features); + assertEquals(direct.length, cached.length); + int bestCached = 0; + int bestDirect = 0; + for (int o = 0; o < cached.length; o++) { + assertEquals(direct[o], cached[o], + Math.max(1.0e-6, Math.abs(direct[o]) * 1.0e-6), + "score " + o + " must agree to float rounding"); + if (cached[o] > cached[bestCached]) { + bestCached = o; + } + if (direct[o] > direct[bestDirect]) { + bestDirect = o; + } + } + assertEquals(bestDirect, bestCached, "the winning transition must be identical"); + } + } + + @Test + void testNormalizeUsesTheUnicodeDataCaseMapping() { + // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's + // String.toLowerCase would render this word as "i" + COMBINING DOT ABOVE instead. + assertEquals(StringUtil.toLowerCase("\u0130STANBUL"), + FeedforwardDependencyModel.normalize("\u0130STANBUL")); + assertEquals("istanbul", FeedforwardDependencyModel.normalize("\u0130STANBUL")); + // special symbols still pass through untouched + assertEquals(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.normalize(FeedforwardDependencyModel.UNKNOWN)); + assertNull(FeedforwardDependencyModel.normalize(null)); + } + + @Test + void testRefineValidation() { + final FeedforwardDependencyTrainer.Settings settings = + FeedforwardDependencyTrainer.Settings.defaults(); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(null, ObjectStreamUtils.createObjectStream(corpus()), settings, 4)); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(model, ObjectStreamUtils.createObjectStream(corpus()), settings, 1)); + } + + @Test + void testBeamSizeValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(model, 0)); + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null, 4)); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + final FeedforwardDependencyModel reloaded = + FeedforwardDependencyModel.load(new ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new FeedforwardDependencyParser(reloaded) + .parse(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), parsed); + } + + @Test + void testCorruptModelFailsLoud() { + assertThrows(IOException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream("not a model".getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void testSettingsValidation() { + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(0, 32, 10, 32, 0.05, 0.0, 0.0, 1, 17L)); + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(16, 32, 10, 32, -1.0, 0.0, 0.0, 1, 17L)); + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(16, 32, 10, 32, 0.05, 0.0, 1.0, 1, 17L)); + } + + @Test + void testPretrainedSeedingAppliesAndValidates() throws IOException { + // near-zero learning keeps the seeded row observable after one epoch + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(4, 8, 1, 32, 1e-9, 0.0, 0.0, 1, 17L); + final float[] vector = {0.25f, -0.5f, 0.75f, -1.0f}; + final FeedforwardDependencyModel seeded = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> "dog".equals(word) ? vector.clone() : null); + final int row = seeded.wordIds().get("dog"); + for (int d = 0; d < vector.length; d++) { + assertEquals(vector[d], seeded.embeddings()[row][d], 1e-4); + } + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> new float[] {1.0f})); + } + + @Test + void testArgumentValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null)); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train(null, + FeedforwardDependencyTrainer.Settings.defaults())); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java new file mode 100644 index 0000000000..a46866855c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the outcome encoding of {@link Transition}: every transition must render to a + * unique outcome string and decode back to an equal transition, and every malformed + * outcome must be rejected loudly. + */ +public class TransitionTest { + + @Test + void testEncodeRendersTheOutcomeStrings() { + assertEquals("SHIFT", Transition.SHIFT.encode()); + assertEquals("LEFT_ARC:nsubj", Transition.leftArc("nsubj").encode()); + assertEquals("RIGHT_ARC:obj", Transition.rightArc("obj").encode()); + } + + @Test + void testDecodeRestoresEncodedTransitions() { + assertSame(Transition.SHIFT, Transition.decode("SHIFT")); + assertEquals(Transition.leftArc("nsubj"), Transition.decode("LEFT_ARC:nsubj")); + assertEquals(Transition.rightArc("obj"), Transition.decode("RIGHT_ARC:obj")); + } + + @Test + void testLabelContainingTheSeparatorRoundTrips() { + // Only the first separator splits type from label, so a label containing the + // separator character itself survives the round trip unchanged. + final Transition transition = Transition.leftArc("nmod:poss"); + assertEquals("LEFT_ARC:nmod:poss", transition.encode()); + assertEquals(transition, Transition.decode(transition.encode())); + } + + @Test + void testDecodeRejectsMalformedOutcomes() { + assertThrows(IllegalArgumentException.class, () -> Transition.decode(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN")); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN:det")); + // A labeled shift is contradictory and must be rejected by the record invariant. + assertThrows(IllegalArgumentException.class, () -> Transition.decode("SHIFT:det")); + } + + @Test + void testConstructorValidation() { + assertThrows(IllegalArgumentException.class, () -> new Transition(null, "det")); + assertThrows(IllegalArgumentException.class, + () -> new Transition(Transition.Type.SHIFT, "det")); + assertThrows(IllegalArgumentException.class, () -> Transition.leftArc(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.rightArc(" ")); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java new file mode 100644 index 0000000000..7af7a75162 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.namefind.NameFinderAnnotator; +import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.postag.POSTagger; +import opennlp.tools.postag.POSTaggerAnnotator; +import opennlp.tools.sentdetect.SentenceDetectorAnnotator; +import opennlp.tools.tokenize.TokenizerAnnotator; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using the + * deterministic components from {@link TestComponents} and a fixed-vocabulary tagger. + * The point under test is the pipeline mechanics and span arithmetic, not model quality. + */ +public class DocumentAnalyzerTest { + + /** Tags the known verbs of the test texts {@code VBZ} and everything else {@code X}. */ + private static final POSTagger TAGGER = new POSTagger() { + + private final Set verbs = Set.of("barks.", "eats."); + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + tags[i] = verbs.contains(sentence[i]) ? "VBZ" : "X"; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException(); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException(); + } + }; + + /** A finder that finds no names, for tests that only exercise the pipeline plumbing. */ + private static final TokenNameFinder NO_NAMES = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + + /** + * Verifies that the adapters identify themselves by their simple class name, which is + * how a pipeline validation message names the offending annotator. + */ + @Test + void testAdaptersNameThemselvesByClassName() { + assertEquals("SentenceDetectorAnnotator", + new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER).toString()); + assertEquals("TokenizerAnnotator", + new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER).toString()); + assertEquals("POSTaggerAnnotator", new POSTaggerAnnotator(TAGGER).toString()); + assertEquals("NameFinderAnnotator", new NameFinderAnnotator(NO_NAMES).toString()); + } + + @Test + void testPipelineProducesAlignedLayersInOriginalCoordinates() { + final Document document = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) + .add(new POSTaggerAnnotator(TAGGER)) + .build() + .analyze("the dog barks. she eats."); + + final List> sentences = document.get(Layers.SENTENCES); + assertEquals(2, sentences.size()); + assertEquals("she eats.", sentences.get(1).value()); + + final List> tokens = document.get(Layers.TOKENS); + assertEquals(5, tokens.size()); + // token of the second sentence, span in document coordinates + assertEquals("she", tokens.get(3).value()); + assertEquals(new Span(15, 18), tokens.get(3).span()); + + final List> tags = document.get(Layers.POS_TAGS); + assertEquals(5, tags.size()); + assertEquals("VBZ", tags.get(2).value()); + assertEquals(tokens.get(2).span(), tags.get(2).span()); + } + + /** + * Verifies that a full pipeline over empty and whitespace-only input produces a + * document on which every provided layer is present and empty, rather than failing: + * zero sentences legitimately yield zero tokens, zero tags, and zero entities. + */ + @ParameterizedTest + @ValueSource(strings = {"", " "}) + void testEmptyAndBlankInputProduceEmptyLayers(String text) { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) + .add(new POSTaggerAnnotator(TAGGER)) + .add(new NameFinderAnnotator(NO_NAMES)) + .build(); + + final Document document = analyzer.analyze(text); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, Layers.ENTITIES), + document.layers()); + for (final LayerKey layer : document.layers()) { + assertTrue(document.get(layer).isEmpty()); + } + } + + /** + * Verifies that a present-but-empty sentence layer is honored as "no sentences": the + * tokenizer adds a present-but-empty token layer instead of tokenizing the whole text. + */ + @Test + void testTokenizerHonorsPresentButEmptySentenceLayer() { + final Document document = new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER) + .annotate(Document.of("the dog").with(Layers.SENTENCES, List.of())); + assertTrue(document.layers().contains(Layers.TOKENS)); + assertTrue(document.get(Layers.TOKENS).isEmpty()); + } + + @Test + void testTokenizerWorksWithoutSentences() { + final Document document = DocumentAnalyzer.builder() + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) + .build() + .analyze("the dog"); + assertEquals(2, document.get(Layers.TOKENS).size()); + } + + @Test + void testMisorderedPipelineFailsAtBuildTime() { + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new POSTaggerAnnotator(TAGGER)); + assertThrows(IllegalArgumentException.class, builder::build); + } + + @Test + void testAnnotatorAdaptersRejectNullDelegates() { + assertThrows(IllegalArgumentException.class, () -> new SentenceDetectorAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new TokenizerAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new POSTaggerAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(null)); + } + + /** + * Verifies that every adapter rejects a {@code null} document with the shared + * message, whether it checks itself or through + * {@link DocumentAnnotators#requireLayers(Document, LayerKey[])}. + */ + @Test + void testAnnotatorAdaptersRejectNullDocuments() { + final List adapters = List.of( + new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER), + new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER), + new POSTaggerAnnotator(TAGGER), + new NameFinderAnnotator(NO_NAMES)); + for (final DocumentAnnotator adapter : adapters) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> adapter.annotate(null)); + assertEquals("document must not be null", e.getMessage()); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java new file mode 100644 index 0000000000..f38737d668 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.postag.POSTagger; +import opennlp.tools.postag.POSTaggerAnnotator; +import opennlp.tools.sentdetect.SentenceDetectorAnnotator; +import opennlp.tools.tokenize.TokenizerAnnotator; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Walks through the document pipeline the way a first-time user would: wrap existing + * analysis components in their adapter annotators, add one custom annotator, build a + * {@link DocumentAnalyzer}, analyze a two-sentence text, and read every layer back with + * its spans in original text coordinates. + * + *

The wrapped components are the tiny deterministic stand-ins from + * {@link TestComponents}, so every expected span and value below follows directly from + * the input text. The point under demonstration is how the layers connect, not the + * quality of any single step.

+ */ +public class DocumentPipelineExampleTest { + + /** + * The key of the custom layer produced by {@link TokenLengthAnnotator}. Any producer + * may introduce such a key in its own code; the container needs no change for it. + */ + private static final LayerKey TOKEN_LENGTHS = + LayerKey.of("token-lengths", Integer.class); + + /** + * A deterministic tagger backed by a fixed dictionary covering exactly the tokens of + * the example text. An unknown token fails the test immediately rather than receiving + * a silent fallback tag. + */ + private static final POSTagger DICTIONARY_TAGGER = new POSTagger() { + + private final Map tagsByToken = Map.of( + "The", "DT", "dog", "NN", "barks.", "VBZ", "It", "PRP", "naps.", "VBZ"); + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + final String tag = tagsByToken.get(sentence[i]); + if (tag == null) { + throw new IllegalArgumentException("no tag defined for token: " + sentence[i]); + } + tags[i] = tag; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + }; + + /** + * A custom pipeline step written directly against {@link DocumentAnnotator}: it reads + * the token layer and provides {@link #TOKEN_LENGTHS}, one annotation per token on the + * token's span, whose value is the character length of the token text. + */ + private static final class TokenLengthAnnotator implements DocumentAnnotator { + + /** + * Adds the {@link #TOKEN_LENGTHS} layer computed from {@link Layers#TOKENS}. + * + * @param document The document to annotate. Must not be {@code null} and must + * contain the token layer, which may be empty. + * @return A new {@link Document} carrying the token length layer. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or the + * token layer is absent. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.TOKENS); + final List> tokens = document.get(Layers.TOKENS); + final List> lengths = new ArrayList<>(tokens.size()); + for (final Annotation token : tokens) { + lengths.add(new Annotation<>(token.span(), token.value().length())); + } + return document.with(TOKEN_LENGTHS, lengths); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(TOKEN_LENGTHS); + } + } + + /** + * Mirrors the manual's document-scoped layer example: a language id rides a + * document-scoped key as a span-less value with exactly the id, value, and null span + * the chapter shows. + */ + @Test + void testDocumentScopedLayerExample() { + final LayerKey language = LayerKey.document("app:language", String.class); + final Document document = Document.of("The dog barks. It naps."); + + final Document tagged = document.with(language, List.of(Annotation.of("eng"))); + assertEquals("eng", tagged.get(language).get(0).value()); + assertNull(tagged.get(language).get(0).span()); + } + + /** + * Runs the full pipeline story: sentences, tokens, part-of-speech tags, and one custom + * layer over a two-sentence text, then verifies every annotation of every layer, span + * by span, in original text coordinates. + */ + @Test + void testFullPipelineStory() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) + .add(new POSTaggerAnnotator(DICTIONARY_TAGGER)) + .add(new TokenLengthAnnotator()) + .build(); + + final Document document = analyzer.analyze("The dog barks. It naps."); + + // The document carries exactly the four layers the pipeline provides. + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, TOKEN_LENGTHS), + document.layers()); + assertEquals("The dog barks. It naps.", document.text()); + + // Sentence layer: one annotation per sentence, covering it in document coordinates. + final List> sentences = document.get(Layers.SENTENCES); + assertEquals(2, sentences.size()); + assertEquals(new Span(0, 14), sentences.get(0).span()); + assertEquals("The dog barks.", sentences.get(0).value()); + assertEquals(new Span(15, 23), sentences.get(1).span()); + assertEquals("It naps.", sentences.get(1).value()); + + // Token layer: five tokens; the second sentence's spans are shifted back to + // document coordinates, so every span can index into the original text. + final List> tokens = document.get(Layers.TOKENS); + assertEquals(5, tokens.size()); + assertEquals(new Span(0, 3), tokens.get(0).span()); + assertEquals("The", tokens.get(0).value()); + assertEquals(new Span(4, 7), tokens.get(1).span()); + assertEquals("dog", tokens.get(1).value()); + assertEquals(new Span(8, 14), tokens.get(2).span()); + assertEquals("barks.", tokens.get(2).value()); + assertEquals(new Span(15, 17), tokens.get(3).span()); + assertEquals("It", tokens.get(3).value()); + assertEquals(new Span(18, 23), tokens.get(4).span()); + assertEquals("naps.", tokens.get(4).value()); + + // Tag layer: aligned with the token layer by position, each tag on its token's span. + final List> tags = document.get(Layers.POS_TAGS); + assertEquals(5, tags.size()); + assertEquals("DT", tags.get(0).value()); + assertEquals("NN", tags.get(1).value()); + assertEquals("VBZ", tags.get(2).value()); + assertEquals("PRP", tags.get(3).value()); + assertEquals("VBZ", tags.get(4).value()); + for (int i = 0; i < tags.size(); i++) { + assertEquals(tokens.get(i).span(), tags.get(i).span()); + } + + // Custom layer: the container returns it as List>, so the + // values are used as numbers without a cast. + final List> lengths = document.get(TOKEN_LENGTHS); + assertEquals(5, lengths.size()); + assertEquals(3, lengths.get(0).value()); + assertEquals(3, lengths.get(1).value()); + assertEquals(6, lengths.get(2).value()); + assertEquals(2, lengths.get(3).value()); + assertEquals(5, lengths.get(4).value()); + for (int i = 0; i < lengths.size(); i++) { + assertEquals(tokens.get(i).span(), lengths.get(i).span()); + } + + // Every span refers to the original text, so covered text always round-trips. + for (final Annotation token : tokens) { + assertEquals(token.value(), + token.span().getCoveredText(document.text()).toString()); + } + } + + /** + * Verifies that the analyzer leaves the input untouched between calls: analyzing two + * texts with the same analyzer yields two independent documents. + */ + @Test + void testAnalyzerIsReusableAcrossTexts() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) + .build(); + + final Document first = analyzer.analyze("The dog barks."); + final Document second = analyzer.analyze("It naps."); + + assertEquals(3, first.get(Layers.TOKENS).size()); + assertEquals(2, second.get(Layers.TOKENS).size()); + assertEquals("The dog barks.", first.text()); + assertEquals("It naps.", second.text()); + assertEquals(1, first.get(Layers.SENTENCES).size()); + assertEquals(1, second.get(Layers.SENTENCES).size()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java new file mode 100644 index 0000000000..6145e5b790 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +/** + * Deterministic stand-in components shared by the document pipeline tests, so every + * expected span in those tests follows directly from the definitions here. + */ +final class TestComponents { + + /** + * A deterministic sentence detector that ends a sentence after every period and + * expects a single space between sentences. Only the span-producing method is + * implemented because the adapter calls no other method. + */ + static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the adapter only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A deterministic tokenizer that splits on single space characters and keeps all + * other characters, including sentence-final periods, attached to their token. Only + * the span-producing method is implemented because the adapter calls no other method. + */ + static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the adapter only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + private TestComponents() { + // Not instantiated; this class provides shared test fixtures only. + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java new file mode 100644 index 0000000000..63654bbada --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class LemmatizerAnnotatorTest { + + /** Lowercases verbs and keeps everything else, enough to observe the adapter. */ + private static final Lemmatizer FIXTURE = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + final String[] lemmas = new String[toks.length]; + for (int i = 0; i < toks.length; i++) { + lemmas[i] = "VERB".equals(tags[i]) ? "run" : toks[i]; + } + return lemmas; + } + + @Override + public List> lemmatize(List toks, List tags) { + throw new UnsupportedOperationException(); + } + }; + + @Test + void testLemmasAlignWithTokens() { + final Document document = Document.of("She ran home") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 12), "She ran home"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "She"), + new Annotation<>(new Span(4, 7), "ran"), + new Annotation<>(new Span(8, 12), "home"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PRON"), + new Annotation<>(new Span(4, 7), "VERB"), + new Annotation<>(new Span(8, 12), "NOUN"))); + + final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); + + final List> lemmas = lemmatized.get(LemmatizerAnnotator.LEMMAS); + assertEquals(3, lemmas.size()); + assertEquals("run", lemmas.get(1).value()); + assertEquals(new Span(4, 7), lemmas.get(1).span()); + assertEquals("home", lemmas.get(2).value()); + } + + /** + * Verifies that the lemmatizer is invoked once per sentence with exactly that + * sentence's tokens and tags, so lemmatization decisions never see material from a + * neighboring sentence, and that the lemma layer still aligns with the token layer. + */ + @Test + void testLemmatizesPerSentence() { + final List> calls = new ArrayList<>(); + final Lemmatizer recording = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + calls.add(List.of(toks)); + return toks.clone(); + } + + @Override + public List> lemmatize(List toks, List tags) { + throw new UnsupportedOperationException(); + } + }; + final Document document = Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PROPN"), + new Annotation<>(new Span(4, 9), "VERB"), + new Annotation<>(new Span(10, 13), "PROPN"), + new Annotation<>(new Span(14, 19), "VERB"))); + + final Document lemmatized = new LemmatizerAnnotator(recording).annotate(document); + + assertEquals(List.of( + List.of("Ana", "runs."), + List.of("Bob", "sits.")), calls); + assertEquals(4, lemmatized.get(LemmatizerAnnotator.LEMMAS).size()); + assertEquals(new Span(10, 13), + lemmatized.get(LemmatizerAnnotator.LEMMAS).get(2).span()); + } + + /** + * Verifies that the adapter identifies itself by its simple class name, which is how a + * pipeline validation message names the offending annotator. + */ + @Test + void testAdapterNamesItselfByClassName() { + assertEquals("LemmatizerAnnotator", new LemmatizerAnnotator(FIXTURE).toString()); + } + + @Test + void testInvalidArguments() { + assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(null)); + final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + final Document misaligned = Document.of("a b") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "a b"))) + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))) + .with(Layers.POS_TAGS, List.of()); + assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(misaligned)); + } + + /** + * Verifies that a document lacking a required layer is rejected with a message naming + * the missing layer, for each of the three required layers in declaration order. + */ + @Test + void testAbsentRequiredLayerThrowsWithExactMessage() { + final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no layers"))); + assertEquals("document lacks the required layer opennlp:sentences", + e.getMessage()); + + final Document sentencesOnly = Document.of("a") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 1), "a"))); + final IllegalArgumentException tokenless = assertThrows( + IllegalArgumentException.class, () -> annotator.annotate(sentencesOnly)); + assertEquals("document lacks the required layer opennlp:tokens", + tokenless.getMessage()); + + final Document untagged = sentencesOnly + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))); + final IllegalArgumentException tagless = assertThrows( + IllegalArgumentException.class, () -> annotator.annotate(untagged)); + assertEquals("document lacks the required layer opennlp:pos", + tagless.getMessage()); + } + + /** + * Verifies that present-but-empty layers yield a present-but-empty lemma layer rather + * than an exception. + */ + @Test + void testEmptyPresentLayersYieldEmptyLemmaLayer() { + final Document document = Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of()); + final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); + assertTrue(lemmatized.layers().contains(LemmatizerAnnotator.LEMMAS)); + assertTrue(lemmatized.get(LemmatizerAnnotator.LEMMAS).isEmpty()); + } + + /** + * Verifies that a token lying outside every sentence is rejected loudly, matching the + * walk contract of the other per-sentence adapters. + */ + @Test + void testTokenOutsideEverySentenceThrowsWithExactMessage() { + final Document document = Document.of("Ana runs. Bob") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PROPN"), + new Annotation<>(new Span(4, 9), "VERB"), + new Annotation<>(new Span(10, 13), "PROPN"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(FIXTURE).annotate(document)); + assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies that a lemmatizer returning a wrong number of lemmas for a sentence is + * rejected loudly instead of silently misaligning the lemma layer. + */ + @Test + void testWrongLemmaCountFailsLoud() { + final Lemmatizer shortLemmatizer = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + return new String[] {"a"}; + } + + @Override + public List> lemmatize(List toks, List tags) { + throw new UnsupportedOperationException(); + } + }; + final Document document = Document.of("a b") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "a b"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 1), "a"), + new Annotation<>(new Span(2, 3), "b"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 1), "X"), + new Annotation<>(new Span(2, 3), "X"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(shortLemmatizer).annotate(document)); + assertEquals("lemmatizer returned 1 lemmas for 2 tokens", e.getMessage()); + } + + /** + * Verifies that the adapter declares all three consumed layers as required, so a + * pipeline without a sentence step fails at build time. + */ + @Test + void testRequiresSentencesTokensAndTags() { + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + new LemmatizerAnnotator(FIXTURE).requires()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderAnnotatorTest.java new file mode 100644 index 0000000000..14f294c8ce --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderAnnotatorTest.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.namefind; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link NameFinderAnnotator} maps token-index mentions to character spans on + * the original text, carries the entity type as the annotation value on an untyped span, + * and clears the finder's adaptive data per document. + */ +public class NameFinderAnnotatorTest { + + /** + * Builds a finder over a fixed find function, with adaptive-data clearing counted + * in the given counter when one is supplied. + * + * @param find Maps a sentence's tokens to its mentions. + * @param cleared Counts {@code clearAdaptiveData} calls, or {@code null} to ignore. + * @return The finder fixture. Never {@code null}. + */ + private static TokenNameFinder finder(Function find, + AtomicInteger cleared) { + return new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return find.apply(tokens); + } + + @Override + public void clearAdaptiveData() { + if (cleared != null) { + cleared.incrementAndGet(); + } + } + }; + } + + /** + * @return A two-sentence document with sentence and token layers over + * {@code "Ana runs. Bob sits."}. Never {@code null}. + */ + private static Document twoSentenceDocument() { + return Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))); + } + + @Test + void testTokenIndexSpansBecomeCharacterSpans() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> { + // "New York" as a two-token location mention + return new Span[] {new Span(1, 3, "location")}; + }, cleared); + + final Document document = Document.of("in New York today") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 17), "in New York today"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "in"), + new Annotation<>(new Span(3, 6), "New"), + new Annotation<>(new Span(7, 11), "York"), + new Annotation<>(new Span(12, 17), "today"))); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + final List> entities = annotated.get(Layers.ENTITIES); + assertEquals(1, entities.size()); + assertEquals(new Span(3, 11), entities.get(0).span()); + // the annotation value is the single source of the entity type; the span is untyped + assertEquals("location", entities.get(0).value()); + assertNull(entities.get(0).span().getType()); + assertEquals("New York", + entities.get(0).span().getCoveredText(annotated.text()).toString()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that a document carrying sentences but no token layer is rejected with a + * message naming the token layer, so the token check is exercised on its own rather + * than being shadowed by the sentence check. + */ + @Test + void testMissingTokenLayerThrows() { + final TokenNameFinder finder = finder(tokens -> new Span[0], null); + final Document document = Document.of("no tokens") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "no tokens"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); + } + + /** + * Verifies that a mention the finder returns without a type is recorded with the + * {@link NameFinderAnnotator#UNTYPED} label, and that the label is the name-sample + * default type, so downstream consumers can rely on the two being interchangeable. + */ + @Test + void testUntypedMentionRecordedAsUntyped() { + final TokenNameFinder finder = finder(tokens -> new Span[] {new Span(0, 1)}, null); + final Document document = Document.of("Ana runs.") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."))); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + final List> entities = annotated.get(Layers.ENTITIES); + assertEquals(1, entities.size()); + assertEquals(NameSample.DEFAULT_TYPE, NameFinderAnnotator.UNTYPED); + assertEquals(NameFinderAnnotator.UNTYPED, entities.get(0).value()); + assertEquals(new Span(0, 3), entities.get(0).span()); + assertNull(entities.get(0).span().getType()); + } + + /** + * Verifies that a mention whose token indices reach beyond its sentence's tokens is + * rejected loudly instead of silently taking its character span from the following + * sentence's tokens, and that the adaptive data is still cleared on that failure. + */ + @Test + void testMentionOutsideSentenceTokensFailsLoud() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> { + // two tokens in the sentence, but the mention claims three + return new Span[] {new Span(0, 3, "person")}; + }, cleared); + final Document document = twoSentenceDocument(); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("finder returned mention [0..3) person outside the sentence's 2 tokens", + e.getMessage()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that a zero-length mention is rejected loudly. {@link Span} permits + * {@code start == end}, but such a mention covers no token, so mapping its end + * through {@code end - 1} would read the previous sentence's last token instead of + * failing. The finder here returns the empty mention for the second sentence, the + * case that would otherwise be mapped silently wrong, and the adaptive data is still + * cleared on the failure. + */ + @Test + void testZeroLengthMentionFailsLoud() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> + "Bob".equals(tokens[0]) ? new Span[] {new Span(0, 0)} : new Span[0], cleared); + final Document document = twoSentenceDocument(); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("finder returned mention [0..0) outside the sentence's 2 tokens", + e.getMessage()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that a token lying outside every sentence is rejected loudly and that the + * adaptive data is still cleared on that failure, so a rejected document cannot leak + * finder state into the next one. + */ + @Test + void testTokenOutsideEverySentenceThrowsAndStillClears() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> new Span[0], cleared); + final Document document = Document.of("Ana runs. Bob") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that the finder is invoked once per sentence with exactly that sentence's + * tokens, that sentence-local mention indices are mapped through the sentence's first + * token position into document character spans, and that the adaptive data is cleared + * exactly once after the whole document. + */ + @Test + void testFindsPerSentenceAndMapsSentenceLocalIndices() { + final List> calls = new ArrayList<>(); + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> { + calls.add(List.of(tokens)); + // the first token of every sentence is a person mention, in sentence-local indices + return new Span[] {new Span(0, 1, "person")}; + }, cleared); + + final Document annotated = new NameFinderAnnotator(finder).annotate(twoSentenceDocument()); + + assertEquals(List.of( + List.of("Ana", "runs."), + List.of("Bob", "sits.")), calls); + + final List> entities = annotated.get(Layers.ENTITIES); + assertEquals(2, entities.size()); + assertEquals(new Span(0, 3), entities.get(0).span()); + assertEquals("person", entities.get(0).value()); + assertEquals(new Span(10, 13), entities.get(1).span()); + assertEquals("person", entities.get(1).value()); + assertEquals("Bob", + entities.get(1).span().getCoveredText(annotated.text()).toString()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that the annotator declares both the sentence layer and the token layer as + * required, so a pipeline without a sentence step fails at build time. + */ + @Test + void testRequiresSentencesAndTokens() { + final TokenNameFinder finder = finder(tokens -> new Span[0], null); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), + new NameFinderAnnotator(finder).requires()); + } + + /** + * Verifies that present-but-empty sentence and token layers yield a present-but-empty + * entity layer without invoking the finder, rather than an exception. + */ + @Test + void testEmptyPresentLayersYieldEmptyEntityLayer() { + final AtomicInteger found = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> { + found.incrementAndGet(); + return new Span[0]; + }, null); + final Document document = Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + + assertTrue(annotated.layers().contains(Layers.ENTITIES)); + assertTrue(annotated.get(Layers.ENTITIES).isEmpty()); + assertEquals(0, found.get()); + } + + /** + * Verifies that a document without a sentence layer is rejected with a message naming + * the missing layer. + */ + @Test + void testAbsentSentenceLayerThrowsWithExactMessage() { + final TokenNameFinder finder = finder(tokens -> new Span[0], null); + final Document document = Document.of("Ana") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "Ana"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("document lacks the required layer opennlp:sentences", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java new file mode 100644 index 0000000000..dee78a2ef7 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.parser.ParserAnnotator.Phrase; +import opennlp.tools.util.Span; + +public class ParserAnnotatorTest { + + /** + * A parser that builds one fixed bracketing over any sentence of six tokens, + * {@code (S (NP (NP 0 1) (PP 2 (NP 3))) (VP 4) 5)}, with explicit heads, so the + * span and head mapping is observable without a model. + */ + private static class FixedParser implements Parser { + + @Override + public Parse[] parse(Parse tokens, int numParses) { + return new Parse[] {parse(tokens)}; + } + + @Override + public Parse parse(Parse tokens) { + final Parse[] toks = tokens.getChildren(); + if (toks.length != 6) { + return tokens; + } + final String[] tags = {"DT", "NN", "IN", "NNP", "VBD", "."}; + final Parse[] pos = new Parse[toks.length]; + for (int i = 0; i < toks.length; i++) { + pos[i] = node(tokens, tags[i], toks[i], toks[i], toks[i]); + tokens.insert(pos[i]); + } + final Parse innerNp = node(tokens, "NP", pos[0], pos[1], toks[1]); + tokens.insert(innerNp); + final Parse maryNp = node(tokens, "NP", pos[3], pos[3], toks[3]); + tokens.insert(maryNp); + final Parse pp = node(tokens, "PP", pos[2], maryNp, toks[2]); + tokens.insert(pp); + final Parse outerNp = node(tokens, "NP", innerNp, pp, toks[1]); + tokens.insert(outerNp); + final Parse vp = node(tokens, "VP", pos[4], pos[4], toks[4]); + tokens.insert(vp); + final Parse s = node(tokens, "S", outerNp, pos[5], toks[4]); + tokens.insert(s); + return tokens; + } + + private static Parse node(Parse root, String type, Parse from, Parse to, Parse head) { + return new Parse(root.getText(), + new Span(from.getSpan().getStart(), to.getSpan().getEnd()), type, 1.0, head); + } + } + + private static List> tokens(String text, String... forms) { + final List> annotations = new ArrayList<>(forms.length); + int cursor = 0; + for (final String form : forms) { + final int start = text.indexOf(form, cursor); + annotations.add(new Annotation<>(new Span(start, start + form.length()), form)); + cursor = start + form.length(); + } + return annotations; + } + + /** One six-token sentence whose text carries a double space the parse text lacks. */ + private static Document sentence() { + final String text = "The dog of Mary ran."; + return Document.of(text) + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 21), "s"))) + .with(Layers.TOKENS, tokens(text, "The", "dog", "of", "Mary", "ran", ".")); + } + + @Test + void testEmitsPhrasesInPreOrderOnTokenSpansWithHeads() { + final Document document = new ParserAnnotator(new FixedParser()).annotate(sentence()); + + final List> phrases = document.get(ParserAnnotator.PHRASES); + Assertions.assertEquals(List.of("S", "NP", "NP", "PP", "NP", "VP"), + phrases.stream().map(a -> a.value().label()).toList()); + Assertions.assertEquals(List.of( + new Span(0, 21), new Span(0, 16), new Span(0, 7), new Span(9, 16), + new Span(12, 16), new Span(17, 20)), + phrases.stream().map(Annotation::span).toList()); + Assertions.assertEquals("The dog of Mary", document.text().subSequence(0, 16).toString()); + final Span dog = new Span(4, 7); + final Span ran = new Span(17, 20); + Assertions.assertEquals(List.of(ran, dog, dog, new Span(9, 11), new Span(12, 16), ran), + phrases.stream().map(a -> a.value().head()).toList()); + } + + @Test + void testEmptyLayersYieldEmptyPhraseLayer() { + final Document document = new ParserAnnotator(new FixedParser()).annotate( + Document.of("").with(Layers.SENTENCES, List.of()).with(Layers.TOKENS, List.of())); + Assertions.assertTrue(document.layers().contains(ParserAnnotator.PHRASES)); + Assertions.assertTrue(document.get(ParserAnnotator.PHRASES).isEmpty()); + } + + @Test + void testLayerContract() { + final ParserAnnotator annotator = new ParserAnnotator(new FixedParser()); + Assertions.assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), annotator.requires()); + Assertions.assertEquals(Set.of(ParserAnnotator.PHRASES), annotator.provides()); + Assertions.assertEquals("opennlp:phrases", ParserAnnotator.PHRASES.id()); + Assertions.assertEquals("ParserAnnotator", annotator.toString()); + } + + @Test + void testRejectsNullParserMissingLayersAndNullParse() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new ParserAnnotator(null)); + final ParserAnnotator annotator = new ParserAnnotator(new FixedParser()); + Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("x"))); + final ParserAnnotator silent = new ParserAnnotator(new FixedParser() { + @Override + public Parse parse(Parse tokens) { + return null; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> silent.annotate(sentence())); + } + + @Test + void testPhraseRejectsBlankLabelAndNullHead() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Phrase(" ", new Span(0, 1))); + Assertions.assertThrows(IllegalArgumentException.class, () -> new Phrase("NP", null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSTaggerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSTaggerAnnotatorTest.java new file mode 100644 index 0000000000..72fea7f524 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSTaggerAnnotatorTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.postag; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link POSTaggerAnnotator} tags one sentence per {@link POSTagger#tag(String[])} + * call, keeps the tag layer aligned with the token layer, distinguishes a present-but-empty + * required layer from an absent one, and rejects tokens outside every sentence. + */ +public class POSTaggerAnnotatorTest { + + /** + * A tagger that records the exact token sequence of every call and answers with one + * {@code "X"} tag per token, so the per-call slicing is observable. Tests override + * {@link #tag(String[])} where a deviant answer is the fixture. + */ + private static class RecordingTagger implements POSTagger { + + private final List> calls = new ArrayList<>(); + + @Override + public String[] tag(String[] sentence) { + calls.add(List.of(sentence)); + final String[] tags = new String[sentence.length]; + Arrays.fill(tags, "X"); + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + } + + /** + * @return A two-sentence document with sentence and token layers over + * {@code "The dog barks. It naps."}. Never {@code null}. + */ + private static Document twoSentenceDocument() { + return Document.of("The dog barks. It naps.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 14), "The dog barks."), + new Annotation<>(new Span(15, 23), "It naps."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(8, 14), "barks."), + new Annotation<>(new Span(15, 17), "It"), + new Annotation<>(new Span(18, 23), "naps."))); + } + + /** + * Verifies that the tagger is invoked once per sentence with exactly that sentence's + * tokens, and that the resulting tag layer stays aligned with the token layer by + * position, each tag on its token's span. + */ + @Test + void testTagsEachSentenceSeparately() { + final RecordingTagger tagger = new RecordingTagger(); + final Document annotated = new POSTaggerAnnotator(tagger).annotate(twoSentenceDocument()); + + assertEquals(List.of( + List.of("The", "dog", "barks."), + List.of("It", "naps.")), tagger.calls); + + final List> tokens = annotated.get(Layers.TOKENS); + final List> tags = annotated.get(Layers.POS_TAGS); + assertEquals(tokens.size(), tags.size()); + for (int i = 0; i < tags.size(); i++) { + assertEquals(tokens.get(i).span(), tags.get(i).span()); + assertEquals("X", tags.get(i).value()); + } + } + + /** + * Verifies that the annotator declares both the sentence layer and the token layer as + * required, so a pipeline without a sentence step fails at build time. + */ + @Test + void testRequiresSentencesAndTokens() { + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), + new POSTaggerAnnotator(new RecordingTagger()).requires()); + } + + /** + * Verifies that present-but-empty sentence and token layers yield a present-but-empty + * tag layer without invoking the tagger, rather than an exception. + */ + @Test + void testEmptyPresentLayersYieldEmptyTagLayer() { + final RecordingTagger tagger = new RecordingTagger(); + final Document document = Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()); + + final Document annotated = new POSTaggerAnnotator(tagger).annotate(document); + + assertTrue(annotated.layers().contains(Layers.POS_TAGS)); + assertTrue(annotated.get(Layers.POS_TAGS).isEmpty()); + assertTrue(tagger.calls.isEmpty()); + } + + /** + * Verifies that a document without a sentence layer is rejected with a message naming + * the missing layer. + */ + @Test + void testAbsentSentenceLayerThrowsWithExactMessage() { + final Document document = Document.of("The dog") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); + assertEquals("document lacks the required layer opennlp:sentences", e.getMessage()); + } + + /** + * Verifies that a document without a token layer is rejected with a message naming the + * missing layer. + */ + @Test + void testAbsentTokenLayerThrowsWithExactMessage() { + final Document document = Document.of("The dog") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 7), "The dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); + assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); + } + + /** + * Verifies that a token whose span no sentence encloses is rejected with a message + * naming the token's span. + */ + @Test + void testTokenOutsideEverySentenceThrowsWithExactMessage() { + final Document document = Document.of("The dog") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "The"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); + assertEquals("token at [4..7) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies that a sentence containing no tokens contributes nothing: the tagger is + * never called with an empty sequence and the tag layer still matches the token layer. + */ + @Test + void testSentenceWithoutTokensContributesNothing() { + final RecordingTagger tagger = new RecordingTagger(); + final Document document = Document.of("The ???") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "???"))) + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "The"))); + + final Document annotated = new POSTaggerAnnotator(tagger).annotate(document); + + assertEquals(List.of(List.of("The")), tagger.calls); + assertEquals(1, annotated.get(Layers.POS_TAGS).size()); + } + + /** + * Verifies that a tagger returning a wrong number of tags for a sentence is rejected + * loudly instead of silently misaligning the tag layer with the token layer. + */ + @Test + void testWrongTagCountFailsLoud() { + // one tag regardless of sentence length, so a two-token sentence trips the check + final POSTagger shortTagger = new RecordingTagger() { + + @Override + public String[] tag(String[] sentence) { + return new String[] {"X"}; + } + }; + final Document document = Document.of("The dog") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 7), "The dog"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(shortTagger).annotate(document)); + assertEquals("tagger returned 1 tags for 2 tokens", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java new file mode 100644 index 0000000000..668acd901c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class StemmerAnnotatorTest { + + @Test + void testStemsAlignWithTokens() { + final Document document = Document.of("running dogs") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 7), "running"), + new Annotation<>(new Span(8, 12), "dogs"))); + + final Document stemmed = new StemmerAnnotator( + new PorterStemmer()).annotate(document); + + final List> stems = stemmed.get(StemmerAnnotator.STEMS); + assertEquals(2, stems.size()); + assertEquals("run", stems.get(0).value()); + assertEquals(new Span(0, 7), stems.get(0).span()); + assertEquals("dog", stems.get(1).value()); + } + + @Test + void testInvalidArguments() { + assertThrows(IllegalArgumentException.class, + () -> new StemmerAnnotator(null)); + final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + } + + /** + * Verifies that a misordered pipeline names the adapter by its simple class name, not + * by its default identity string. + */ + @Test + void testPipelineValidationNamesTheAdapter() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnalyzer.builder() + .add(new StemmerAnnotator(new PorterStemmer())).build()); + assertEquals("annotator StemmerAnnotator requires layer opennlp:tokens," + + " which no earlier annotator provides", e.getMessage()); + } + + /** + * Verifies that a document without a token layer is rejected with a message naming the + * missing layer, instead of silently producing an empty stem layer. + */ + @Test + void testAbsentTokenLayerThrowsWithExactMessage() { + final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no tokens"))); + assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); + } + + /** + * Verifies that a present-but-empty token layer yields a present-but-empty stem layer + * rather than an exception. + */ + @Test + void testEmptyPresentTokenLayerYieldsEmptyStemLayer() { + final Document document = Document.of("").with(Layers.TOKENS, List.of()); + final Document stemmed = new StemmerAnnotator(new PorterStemmer()).annotate(document); + assertTrue(stemmed.layers().contains(StemmerAnnotator.STEMS)); + assertTrue(stemmed.get(StemmerAnnotator.STEMS).isEmpty()); + } +} diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml new file mode 100644 index 0000000000..a4e8d31f38 --- /dev/null +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -0,0 +1,83 @@ + + + + + + + Dependency Parsing + +
+ Introduction + + Dependency parsing assigns each token a syntactic head and a relation label. + DependencyParserME trains on DependencySample streams + and returns a DependencyGraph of heads and relations. CoNLL-U + treebanks are read through ConlluDependencySampleStream. + +
+ +
+ Training and parsing + + Open a CoNLL-U sample stream, train a model, and parse tokens with their + part-of-speech tags: + + samples = + new ConlluDependencySampleStream(conlluInput, ConlluTagset.U); + +TrainingParameters parameters = TrainingParameters.defaultParams(); +parameters.put(Parameters.CUTOFF_PARAM, 0); +DependencyModel model = DependencyParserME.train("eng", samples, parameters); +DependencyParserME parser = new DependencyParserME(model); + +DependencyGraph graph = parser.parse( + new String[] {"the", "dog", "barks"}, + new String[] {"DET", "NOUN", "VERB"}); +// graph.headOf(0) == 1, graph.relationOf(0) == "det" +// graph.root() is the index of the root token]]> + + A trained model serializes like any other tool model. Evaluation accumulates + unlabeled and labeled attachment scores through + DependencyEvaluator. + +
+ +
+ DependencyAnnotator + + DependencyAnnotator adapts a DependencyParser to + the document pipeline. It requires sentence, token, and POS-tag layers, + parses each sentence separately, and shifts sentence-local arcs so every + head and dependent is an index into the document-wide token layer. Each + arc annotation sits on its dependent token's span in original text + coordinates. The required layers must be present, but they may be empty: + a document without content yields a present-but-empty arc layer. + DependencyAnnotatorPipelineTest asserts the behavior shown here. + +> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); +// one arc per token; second-sentence heads are document-wide indices +// "she" is token 3 with head 4 ("eats"), span [15..18)]]> + + +
+ diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml new file mode 100644 index 0000000000..76c626ada5 --- /dev/null +++ b/opennlp-docs/src/docbkx/document.xml @@ -0,0 +1,226 @@ + + + + + + + Document Annotation Container + +
+ Introduction + + The package opennlp.tools.document provides an immutable container + that carries the original text of one document together with any number of + annotation layers over it. A layer is a list of Annotation values, + each pairing a Span with a value. A span's offsets count Java + chars (UTF-16 units, so a supplementary-plane character counts as two) from + the beginning of the text exactly as the caller supplied it, never a + normalized or derived form, so every annotation can be highlighted in + the source text. The value's Java type is part of the layer's identity: a token + layer reads back as List<Annotation<String>>, for + example the token The paired with the span [0..3). + + + A document with three annotation layers + +text: The dog barks. It naps. + 0 1 2 + 01234567890123456789012 + +sentences: [0..14) "The dog barks." [15..23) "It naps." +tokens: [0..3) "The" [4..7) "dog" [8..14) "barks." [15..17) "It" [18..23) "naps." +pos-tags: [0..3) "DT" [4..7) "NN" [8..14) "VBZ" [15..17) "PRP" [18..23) "VBZ" + + + Each layer in the figure above is identified by a LayerKey that + names the layer and declares the type of its values: + Layers.TOKENS is a LayerKey<String>, so + document.get(Layers.TOKENS) returns the token annotations with + string values. Two keys are equal when their id, their value type, and their + scope are equal, so independently created constants for the same layer + interoperate. The keys of the core linguistic layers live in + Layers; a capability-specific layer's key lives on the annotator + that provides it, and adding a capability never changes the container. + + + Key ids are namespaced. Keys OpenNLP defines start with the + opennlp: prefix; a user extension must use its own prefix. A + bare id, like the token-lengths key below, is legal for an + application-local layer. Toolkit keys are created through + Layers.key and Layers.documentKey, which apply the + prefix. + + + A document is never modified in place: adding a layer returns a new document + that shares the unchanged layers with its ancestor. Documents created through + Document.of capture their text at construction and are safe to + share between threads. Annotations reference other annotations by index within + their layer, never by object identity. Three invariants make those index + references sound: a layer keeps its insertion order, a layer is immutable once + added and detached from the caller's input list, and providing a layer that + already exists is rejected, so an index reference stays valid for the lifetime + of the document. + + + A key declares its scope. Keys are positional by default: every annotation + carries a span, and consumers never null-check it. A document-scoped key + carries whole-document values without spans, which is where a language id, + a category distribution, or provenance belongs: + + LANGUAGE = LayerKey.document("app:language", String.class); + +Document tagged = document.with(LANGUAGE, List.of(Annotation.of("eng"))); +String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]> + + + A corpus may carry a hand-annotated version of a layer beside a produced one. + The convention is a gold: id prefix on the same key scheme, for + example gold:opennlp:tokens beside opennlp:tokens; + since adding a layer is once-only, competing versions always live under + distinct keys and never replace each other. + +
+ +
+ Building a pipeline + + A pipeline step implements DocumentAnnotator: it reads layers from a + document and returns a new document with its own layers added. An annotator + declares the layers it requires and provides, and a + DocumentAnalyzer validates those declarations when the pipeline is + assembled: every required layer must be provided by an earlier annotator, and no + two annotators may provide the same layer, so a misordered pipeline fails at + build time rather than midway through a document. + + + Adapters for the toolkit's own components are provided: + SentenceDetectorAnnotator, TokenizerAnnotator, + POSTaggerAnnotator, NameFinderAnnotator, + ChunkerAnnotator, ParserAnnotator, + LemmatizerAnnotator, and StemmerAnnotator. Each wraps + one of the existing component APIs; an application that needs only one task + can keep using that component's API directly. The following pipeline combines + three adapters with one custom annotator and analyzes the text from the + introduction's figure; the components behind the adapters are any + SentenceDetector, Tokenizer, and + POSTagger, for example the ME implementations loaded from models: + + + + + DocumentPipelineExampleTest asserts the pipeline and layer + round-trip shown here. + + + The resulting document carries exactly the four layers the pipeline provides: + the figure's three plus the custom token-lengths layer. The sentence + layer holds two annotations, [0..14) covering + The dog barks. and [15..23) covering + It naps.. The token layer holds five tokens whose spans refer to + the document text even inside the second sentence, so + It is [15..17) and naps. is + [18..23). Layers produced per token stay aligned with the token + layer by position: + + > tokens = document.get(Layers.TOKENS); +List> tags = document.get(Layers.POS_TAGS); +for (int i = 0; i < tags.size(); i++) { + // each tag sits on its token's span, e.g. "DT" on [0..3) for "The" + Span span = tags.get(i).span(); +} + +// every span refers to the original text, so covered text round-trips +for (Annotation token : tokens) { + CharSequence covered = token.span().getCoveredText(document.text()); +}]]> + + + Because a document is immutable, independent pipelines can process the same + text in parallel, each starting from its own Document.of(text), + and their results can be joined afterwards: merge returns a new + document carrying the layers of both. The texts must match, and by default a + layer key present on both documents is rejected, so two branches that both + run a tokenizer collide on opennlp:tokens. When the branches + rebuild a shared prefix identically, passing + DuplicateLayerPolicy.KEEP_EQUAL keeps one copy of each agreeing + layer; copies whose contents differ are still rejected. + +
+ +
+ Writing a custom annotator + + The annotator below defines its own key in its own code, reads the token + layer, and provides one integer annotation per token; the analyzer's + build-time validation guarantees a tokenizer ran earlier: + + TOKEN_LENGTHS = + LayerKey.of("token-lengths", Integer.class); + +class TokenLengthAnnotator implements DocumentAnnotator { + + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.TOKENS); + List> tokens = document.get(Layers.TOKENS); + List> lengths = new ArrayList<>(tokens.size()); + for (Annotation token : tokens) { + lengths.add(new Annotation<>(token.span(), token.value().length())); + } + return document.with(TOKEN_LENGTHS, lengths); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(TOKEN_LENGTHS); + } +}]]> + + + Because the key declares the value type, the values come back as numbers + without a cast; for the text above the five values are + 3, 3, 6, 2, 5, each on its token's span: + + > lengths = document.get(TOKEN_LENGTHS); +int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> + + + Required layers must be present, but they may be empty: an empty token layer + yields the annotator's provided layers present but empty, so a pipeline degrades + gracefully on documents without content. An absent required layer is rejected + with an IllegalArgumentException naming the layer, because a + missing pipeline stage is an assembly error, not an empty document. + DocumentAnnotators.requireLayers performs exactly that rejection, + including the null check on the document, and is what the toolkit's own + adapters use. + +
+ diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 36641c2c89..4493b16d7d 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -109,9 +109,11 @@ under the License. + + diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java index a47306d3d4..1951c114a0 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java @@ -679,4 +679,31 @@ void testLowercaseBeyondBMP() { String lc = StringUtil.toLowerCase(input); Assertions.assertArrayEquals(expectedCodePoints, lc.codePoints().toArray()); } + + /** + * Verifies the accepting side of the blank check against the toolkit's whitespace + * definition: empty and JDK-whitespace values are blank, and so are the no-break + * spaces U+00A0 and U+2007, which {@link String#isBlank()} does not cover. + */ + @ParameterizedTest + @ValueSource(strings = {"", " \t\n", "\u00A0", " \u00A0\u2007 "}) + void testIsBlankAcceptsWhitespaceOnlyValues(String input) { + Assertions.assertTrue(StringUtil.isBlank(input)); + } + + /** + * Verifies the rejecting side of the blank check: any non-whitespace code point + * makes a value non-blank, including the supplementary-plane letter U+10428, which + * must be read as one code point rather than two chars. + */ + @ParameterizedTest + @ValueSource(strings = {"a", " a ", "\uD801\uDC28"}) + void testIsBlankRejectsValuesWithContent(String input) { + Assertions.assertFalse(StringUtil.isBlank(input)); + } + + @Test + void testIsBlankWithNullString() { + Assertions.assertThrows(NullPointerException.class, () -> StringUtil.isBlank(null)); + } }