From 49b9612dfcb0651482cb85629cddddec12457201 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 14 Jul 2026 21:27:22 -0400 Subject: [PATCH 01/92] OPENNLP-1888: Document annotation container: typed offset-anchored layers over the original text Adds opennlp.tools.document to opennlp-api: Document (immutable, copy-on-add layer container over the original text), Annotation (a typed value on a Span), LayerKey (open, typed layer identity), and DocumentAnnotator (pipeline step declaring the layers it requires and provides). DocumentAnalyzer assembles annotators into a pipeline validated at build time. Standard keys in Layers cover sentences, tokens, part-of-speech tags, and entities, populated through thin adapters over the existing SentenceDetector, Tokenizer, POSTagger, and TokenNameFinder interfaces, which stay the primary API for single-task use and are unchanged. All spans refer to the text as supplied. No new dependencies. --- .../opennlp/tools/document/Annotation.java | 55 +++++ .../java/opennlp/tools/document/Document.java | 87 ++++++++ .../tools/document/DocumentAnalyzer.java | 116 +++++++++++ .../tools/document/DocumentAnnotator.java | 59 ++++++ .../tools/document/ImmutableDocument.java | 107 ++++++++++ .../java/opennlp/tools/document/LayerKey.java | 102 +++++++++ .../java/opennlp/tools/document/Layers.java | 55 +++++ .../tools/document/NameFinderAnnotator.java | 88 ++++++++ .../tools/document/POSTaggerAnnotator.java | 80 ++++++++ .../document/SentenceDetectorAnnotator.java | 70 +++++++ .../tools/document/TokenizerAnnotator.java | 84 ++++++++ .../tools/document/DocumentAnalyzerTest.java | 194 ++++++++++++++++++ .../opennlp/tools/document/DocumentTest.java | 125 +++++++++++ .../document/NameFinderAnnotatorTest.java | 87 ++++++++ 14 files changed, 1309 insertions(+) create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/Annotation.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/Document.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/Layers.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java new file mode 100644 index 0000000000..eaed6bf7f4 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import opennlp.tools.util.Span; + +/** + * One annotation of a {@link Document}: a typed value anchored to a {@link Span} of the + * document's original text. + * + *

The span always refers to the text the document was created with, never to a + * normalized or otherwise derived form, so any annotation can be highlighted in what the + * caller supplied. Annotations that need to reference other annotations, for example a + * dependency arc naming its head token, do so by the index of the target annotation + * within its layer, never by object identity.

+ * + * @param span The location of the annotation in the original text. Must not be + * {@code null}. + * @param value The annotation value. Must not be {@code null}. + * @param The type of the annotation value. + * + * @since 3.0.0 + */ +public record Annotation(Span span, T value) { + + /** + * Validates the annotation. + * + * @throws IllegalArgumentException Thrown if {@code span} or {@code value} is + * {@code null}. + */ + public Annotation { + if (span == null) { + throw new IllegalArgumentException("span must not be null"); + } + if (value == null) { + throw new IllegalArgumentException("value must not be null"); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java new file mode 100644 index 0000000000..53453ff856 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +/** + * An immutable, offset-anchored annotation container: the original text of one document + * plus any number of typed annotation layers over it. + * + *

A layer is a list of {@link Annotation annotations} identified by a + * {@link LayerKey}. The container itself knows nothing about specific layers; every + * analysis capability contributes its results as one more layer without any change to + * this interface, which is what keeps new capabilities additive. All spans refer to + * {@link #text()} as supplied, never to a derived form.

+ * + *

Documents are immutable: {@link #with(LayerKey, List)} returns a new document that + * shares the unchanged layers. Instances are safe to share between threads.

+ * + * @since 3.0.0 + */ +public interface Document { + + /** + * Creates an empty {@link Document} over a text. + * + * @param text The original document text. Must not be {@code null}. + * @return A {@link Document} without any layers. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static Document of(CharSequence text) { + return ImmutableDocument.empty(text); + } + + /** + * @return The original text of the document. Never {@code null}. + */ + CharSequence text(); + + /** + * Retrieves the annotations of one layer. + * + * @param layer The layer to read. Must not be {@code null}. + * @param The type of the layer's annotation values. + * @return The layer's annotations in their layer order, or an empty list when the + * layer is absent. Never {@code null}; the list is unmodifiable. + * @throws IllegalArgumentException Thrown if {@code layer} is {@code null}. + */ + List> get(LayerKey layer); + + /** + * @return The keys of all layers present on the document. Never {@code null}; the set + * is unmodifiable. + */ + Set> layers(); + + /** + * Returns a new document with one layer added. + * + * @param layer The key of the layer to add. Must not be {@code null} and must not + * already be present. + * @param annotations The annotations of the layer. Must not be {@code null}, must not + * contain {@code null}, every value must be assignable to the + * layer's type, and every span must lie within the text bounds. + * @param The type of the layer's annotation values. + * @return A new {@link Document} sharing this document's text and existing layers. + * Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + Document with(LayerKey layer, List> annotations); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java new file mode 100644 index 0000000000..4ca353ca1c --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Runs a fixed sequence of {@link DocumentAnnotator annotators} over a text, producing + * one {@link Document} that carries every step's layers. + * + *

The pipeline is validated at build time: every annotator's required layers must be + * provided by an earlier annotator, so a misordered pipeline fails when it is assembled + * rather than midway through a document. The analyzer holds no per-call state; it is as + * thread-safe as the annotators it is built from.

+ * + * @since 3.0.0 + */ +public final class DocumentAnalyzer { + + private final List annotators; + + private DocumentAnalyzer(List annotators) { + this.annotators = annotators; + } + + /** + * @return A new {@link Builder}. Never {@code null}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Analyzes a text by running every annotator in order. + * + * @param text The original document text. Must not be {@code null}. + * @return The annotated {@link Document}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + public Document analyze(CharSequence text) { + Document document = Document.of(text); + for (final DocumentAnnotator annotator : annotators) { + document = annotator.annotate(document); + } + return document; + } + + /** + * Assembles a {@link DocumentAnalyzer} from annotators in execution order. + */ + public static final class Builder { + + private final List annotators = new ArrayList<>(); + + private Builder() { + } + + /** + * Appends an annotator to the pipeline. + * + * @param annotator The annotator to run after the ones already added. Must not be + * {@code null}. + * @return This {@link Builder}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code annotator} is {@code null}. + */ + public Builder add(DocumentAnnotator annotator) { + if (annotator == null) { + throw new IllegalArgumentException("annotator must not be null"); + } + annotators.add(annotator); + return this; + } + + /** + * Validates the pipeline and builds the analyzer. + * + * @return A {@link DocumentAnalyzer}. Never {@code null}. + * @throws IllegalArgumentException Thrown if the pipeline is empty or an annotator + * requires a layer no earlier annotator provides. + */ + public DocumentAnalyzer build() { + if (annotators.isEmpty()) { + throw new IllegalArgumentException("a pipeline needs at least one annotator"); + } + final Set> available = new HashSet<>(); + for (final DocumentAnnotator annotator : annotators) { + for (final LayerKey required : annotator.requires()) { + if (!available.contains(required)) { + throw new IllegalArgumentException("annotator " + annotator + + " requires layer " + required + ", which no earlier annotator provides"); + } + } + available.addAll(annotator.provides()); + } + return new DocumentAnalyzer(List.copyOf(annotators)); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java new file mode 100644 index 0000000000..9ea2c65a4d --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Set; + +/** + * A pipeline step that reads layers from a {@link Document} and returns a new document + * with its own layers added. + * + *

An annotator declares the layers it {@link #requires()} and {@link #provides()}, so + * a {@link DocumentAnalyzer} can validate a pipeline before running it. Annotators are + * usually thin adapters over an existing analysis component and should hold no per-call + * state, so one instance can serve concurrent pipelines when the wrapped component + * allows it.

+ * + * @since 3.0.0 + */ +public interface DocumentAnnotator { + + /** + * Annotates a document. + * + * @param document The document to annotate. Must not be {@code null} and must contain + * every layer named by {@link #requires()}. + * @return A new {@link Document} carrying the layers named by {@link #provides()} in + * addition to the input layers. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or lacks + * a required layer. + */ + Document annotate(Document document); + + /** + * @return The keys of the layers this annotator reads. Never {@code null}. + */ + default Set> requires() { + return Set.of(); + } + + /** + * @return The keys of the layers this annotator adds. Never {@code null}. + */ + Set> provides(); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java new file mode 100644 index 0000000000..2ec407ee3e --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.util.Span; + +/** + * The default {@link Document} implementation: an unmodifiable map from layer key to an + * unmodifiable annotation list. Adding a layer copies the map, not the layers, so + * documents grown from a common ancestor share their layer lists. + */ +final class ImmutableDocument implements Document { + + private final CharSequence text; + private final Map, List>> layers; + + private ImmutableDocument(CharSequence text, Map, List>> layers) { + this.text = text; + this.layers = layers; + } + + /** + * Creates a document without any layers. + * + * @param text The original document text. Must not be {@code null}. + * @return An empty {@link ImmutableDocument}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static ImmutableDocument empty(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + return new ImmutableDocument(text, Collections.emptyMap()); + } + + @Override + public CharSequence text() { + return text; + } + + @Override + @SuppressWarnings("unchecked") + public List> get(LayerKey layer) { + if (layer == null) { + throw new IllegalArgumentException("layer must not be null"); + } + final List> annotations = layers.get(layer); + if (annotations == null) { + return List.of(); + } + // safe: with(LayerKey, List) verified every value against the key's type on insertion + return (List>) (List) annotations; + } + + @Override + public Set> layers() { + return Collections.unmodifiableSet(layers.keySet()); + } + + @Override + public Document with(LayerKey layer, List> annotations) { + if (layer == null || annotations == null) { + throw new IllegalArgumentException("layer and annotations must not be null"); + } + if (layers.containsKey(layer)) { + throw new IllegalArgumentException("layer is already present: " + layer); + } + for (final Annotation annotation : annotations) { + if (annotation == null) { + throw new IllegalArgumentException("annotations must not contain null: " + layer); + } + if (!layer.type().isInstance(annotation.value())) { + throw new IllegalArgumentException("value of type " + + annotation.value().getClass().getName() + " does not match layer " + layer); + } + final Span span = annotation.span(); + if (span.getEnd() > text.length()) { + throw new IllegalArgumentException("span " + span + " exceeds the text length " + + text.length() + " in layer " + layer); + } + } + final Map, List>> grown = new LinkedHashMap<>(layers); + grown.put(layer, List.copyOf(annotations)); + return new ImmutableDocument(text, Collections.unmodifiableMap(grown)); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java new file mode 100644 index 0000000000..458a61b4c4 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Objects; + +/** + * Identifies one annotation layer of a {@link Document} and carries the type of that + * layer's annotation values, so reading a layer back is statically typed. + * + *

The key space is deliberately open: any producer may define new keys in its own + * package, and the container never enumerates them. Two keys are equal when both their + * id and their value type are equal, so independently created constants for the same + * layer interoperate. Standard keys for the toolkit's own results live in + * {@link Layers}.

+ * + * @param The type of the annotation values stored under this key. + * + * @since 3.0.0 + */ +public final class LayerKey { + + private final String id; + private final Class type; + + private LayerKey(String id, Class type) { + this.id = id; + this.type = type; + } + + /** + * Creates a {@link LayerKey}. + * + * @param id The layer identifier, for example {@code tokens}. Must not be {@code null} + * or blank. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + public static LayerKey of(String id, Class type) { + if (id == null || id.isBlank()) { + throw new IllegalArgumentException("id must not be null or blank"); + } + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + return new LayerKey<>(id, type); + } + + /** + * @return The layer identifier. Never {@code null}. + */ + public String id() { + return id; + } + + /** + * @return The class of the annotation values stored under this key. Never {@code null}. + */ + public Class type() { + return type; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LayerKey other)) { + return false; + } + return id.equals(other.id) && type.equals(other.type); + } + + @Override + public int hashCode() { + return Objects.hash(id, type); + } + + @Override + public String toString() { + return id + '<' + type.getSimpleName() + '>'; + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java new file mode 100644 index 0000000000..9ff47cd2c3 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +/** + * The standard {@link LayerKey layer keys} for the results the toolkit produces itself. + * + *

This class is a convenience, not a registry: the key space stays open, and any + * producer may define further keys in its own package. New capabilities must never + * require an addition here to function.

+ * + * @since 3.0.0 + */ +public final class Layers { + + /** + * Sentence boundaries; each annotation covers one sentence and carries its text. + */ + public static final LayerKey SENTENCES = LayerKey.of("sentences", String.class); + + /** + * Token boundaries; each annotation covers one token and carries its text. + */ + public static final LayerKey TOKENS = LayerKey.of("tokens", String.class); + + /** + * Part-of-speech tags; one annotation per token, aligned with {@link #TOKENS} by + * position, carrying the tag. + */ + public static final LayerKey POS_TAGS = LayerKey.of("pos", String.class); + + /** + * Named entities; each annotation covers one mention and carries the entity type. + */ + public static final LayerKey ENTITIES = LayerKey.of("entities", String.class); + + private Layers() { + // constants only + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java new file mode 100644 index 0000000000..f5bfecdaf8 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link TokenNameFinder} to the document pipeline: reads {@link Layers#TOKENS}, + * maps the finder's token-index spans to character spans on the original text, and + * provides {@link Layers#ENTITIES} carrying the entity type. + * + *

The finder's adaptive data is cleared after each document, so document order does + * not leak between pipeline runs.

+ * + * @since 3.0.0 + */ +public class NameFinderAnnotator implements DocumentAnnotator { + + private final TokenNameFinder finder; + + /** + * Initializes the adapter. + * + * @param finder The name finder to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code finder} is {@code null}. + */ + public NameFinderAnnotator(TokenNameFinder finder) { + if (finder == null) { + throw new IllegalArgumentException("finder must not be null"); + } + this.finder = finder; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final String[] words = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + } + final List> entities = new ArrayList<>(); + for (final Span mention : finder.find(words)) { + final int start = tokens.get(mention.getStart()).span().getStart(); + final int end = tokens.get(mention.getEnd() - 1).span().getEnd(); + final String type = mention.getType() == null ? "default" : mention.getType(); + entities.add(new Annotation<>(new Span(start, end, type), type)); + } + finder.clearAdaptiveData(); + return document.with(Layers.ENTITIES, entities); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(Layers.ENTITIES); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java new file mode 100644 index 0000000000..1234b6803d --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.postag.POSTagger; + +/** + * Adapts a {@link POSTagger} to the document pipeline: reads {@link Layers#TOKENS} and + * provides {@link Layers#POS_TAGS}, one tag annotation per token on the token's span. + * + * @since 3.0.0 + */ +public class POSTaggerAnnotator implements DocumentAnnotator { + + private final POSTagger tagger; + + /** + * Initializes the adapter. + * + * @param tagger The tagger to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code tagger} is {@code null}. + */ + public POSTaggerAnnotator(POSTagger tagger) { + if (tagger == null) { + throw new IllegalArgumentException("tagger must not be null"); + } + this.tagger = tagger; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final String[] words = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + } + final String[] tags = tagger.tag(words); + final List> tagAnnotations = new ArrayList<>(tags.length); + for (int i = 0; i < tags.length; i++) { + tagAnnotations.add(new Annotation<>(tokens.get(i).span(), tags[i])); + } + return document.with(Layers.POS_TAGS, tagAnnotations); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(Layers.POS_TAGS); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java new file mode 100644 index 0000000000..47e7c58c34 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link SentenceDetector} to the document pipeline: provides + * {@link Layers#SENTENCES} from the document text. + * + *

The wrapped detector stays the primary API for single-task use; this adapter calls + * it like any other caller would.

+ * + * @since 3.0.0 + */ +public class SentenceDetectorAnnotator implements DocumentAnnotator { + + private final SentenceDetector detector; + + /** + * Initializes the adapter. + * + * @param detector The sentence detector to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code detector} is {@code null}. + */ + public SentenceDetectorAnnotator(SentenceDetector detector) { + if (detector == null) { + throw new IllegalArgumentException("detector must not be null"); + } + this.detector = detector; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final CharSequence text = document.text(); + final List> sentences = new ArrayList<>(); + for (final Span span : detector.sentPosDetect(text)) { + sentences.add(new Annotation<>(span, span.getCoveredText(text).toString())); + } + return document.with(Layers.SENTENCES, sentences); + } + + @Override + public Set> provides() { + return Set.of(Layers.SENTENCES); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java new file mode 100644 index 0000000000..625410a181 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link Tokenizer} to the document pipeline: provides {@link Layers#TOKENS}. + * + *

When {@link Layers#SENTENCES} is present, each sentence is tokenized separately and + * the token spans are shifted back to document coordinates; otherwise the whole text is + * tokenized at once. Either way, every token span refers to the original document + * text.

+ * + * @since 3.0.0 + */ +public class TokenizerAnnotator implements DocumentAnnotator { + + private final Tokenizer tokenizer; + + /** + * Initializes the adapter. + * + * @param tokenizer The tokenizer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokenizer} is {@code null}. + */ + public TokenizerAnnotator(Tokenizer tokenizer) { + if (tokenizer == null) { + throw new IllegalArgumentException("tokenizer must not be null"); + } + this.tokenizer = tokenizer; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final String text = document.text().toString(); + final List> tokens = new ArrayList<>(); + final List> sentences = document.get(Layers.SENTENCES); + if (sentences.isEmpty()) { + addTokens(tokens, text, 0); + } else { + for (final Annotation sentence : sentences) { + final Span span = sentence.span(); + addTokens(tokens, text.substring(span.getStart(), span.getEnd()), span.getStart()); + } + } + return document.with(Layers.TOKENS, tokens); + } + + private void addTokens(List> tokens, String text, int offset) { + for (final Span span : tokenizer.tokenizePos(text)) { + final Span shifted = new Span(span.getStart() + offset, span.getEnd() + offset); + tokens.add(new Annotation<>(shifted, span.getCoveredText(text).toString())); + } + } + + @Override + public Set> provides() { + return Set.of(Layers.TOKENS); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java new file mode 100644 index 0000000000..534fe115c4 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.postag.POSTagger; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using simple + * inline implementations of the task interfaces: whitespace tokenization, period sentence + * splitting, and a dictionary tagger. The point under test is the pipeline mechanics and + * span arithmetic, not model quality. + */ +public class DocumentAnalyzerTest { + + private static final SentenceDetector SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException(); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + // split after each period; keep it simple for the test + final String text = s.toString(); + int start = 0; + final java.util.List spans = new java.util.ArrayList<>(); + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + private static final Tokenizer WHITESPACE = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException(); + } + + @Override + public Span[] tokenizePos(String s) { + final java.util.List spans = new java.util.ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + private static final POSTagger TAGGER = new POSTagger() { + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + tags[i] = "barks.".contains(sentence[i]) ? "VBZ" : "X"; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public opennlp.tools.util.Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException(); + } + + @Override + public opennlp.tools.util.Sequence[] topKSequences(String[] sentence, + Object[] additionalContext) { + throw new UnsupportedOperationException(); + } + }; + + @Test + void testPipelineProducesAlignedLayersInOriginalCoordinates() { + final Document document = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(SPLITTER)) + .add(new TokenizerAnnotator(WHITESPACE)) + .add(new POSTaggerAnnotator(TAGGER)) + .build() + .analyze("the dog barks. she eats."); + + final List> sentences = document.get(Layers.SENTENCES); + assertEquals(2, sentences.size()); + assertEquals("she eats.", sentences.get(1).value()); + + final List> tokens = document.get(Layers.TOKENS); + assertEquals(5, tokens.size()); + // token of the second sentence, span in document coordinates + assertEquals("she", tokens.get(3).value()); + assertEquals(new Span(15, 18), tokens.get(3).span()); + + final List> tags = document.get(Layers.POS_TAGS); + assertEquals(5, tags.size()); + assertEquals("VBZ", tags.get(2).value()); + assertEquals(tokens.get(2).span(), tags.get(2).span()); + } + + @Test + void testTokenizerWorksWithoutSentences() { + final Document document = DocumentAnalyzer.builder() + .add(new TokenizerAnnotator(WHITESPACE)) + .build() + .analyze("the dog"); + assertEquals(2, document.get(Layers.TOKENS).size()); + } + + @Test + void testMisorderedPipelineFailsAtBuildTime() { + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new POSTaggerAnnotator(TAGGER)); + assertThrows(IllegalArgumentException.class, builder::build); + } + + @Test + void testEmptyPipelineThrows() { + assertThrows(IllegalArgumentException.class, () -> DocumentAnalyzer.builder().build()); + } + + @Test + void testCustomLayerNeedsNoContainerChange() { + // the additive claim: a brand-new layer type works without touching the container + record Sentiment(String polarity, double score) { + } + final LayerKey sentiment = LayerKey.of("sentiment", Sentiment.class); + final DocumentAnnotator annotator = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + final Span all = new Span(0, document.text().length()); + return document.with(sentiment, + List.of(new Annotation<>(all, new Sentiment("positive", 0.9d)))); + } + + @Override + public Set> provides() { + return Set.of(sentiment); + } + }; + final Document document = DocumentAnalyzer.builder().add(annotator).build() + .analyze("good dog"); + assertEquals("positive", document.get(sentiment).get(0).value().polarity()); + } + + @Test + void testAnnotatorAdaptersRejectNullDelegates() { + assertThrows(IllegalArgumentException.class, () -> new SentenceDetectorAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new TokenizerAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new POSTaggerAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java new file mode 100644 index 0000000000..5af4908bc8 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link Document} container: typed layer access, copy-on-add immutability, + * and the insertion-time validation that protects the layer invariants. + */ +public class DocumentTest { + + private static final LayerKey WORDS = LayerKey.of("words", String.class); + private static final LayerKey NUMBERS = LayerKey.of("numbers", Integer.class); + + @Test + void testEmptyDocument() { + final Document document = Document.of("the dog"); + assertEquals("the dog", document.text()); + assertTrue(document.layers().isEmpty()); + assertTrue(document.get(WORDS).isEmpty()); + } + + @Test + void testWithAddsATypedLayer() { + final Document document = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + assertEquals(Set.of(WORDS), document.layers()); + final List> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals("dog", words.get(1).value()); + assertEquals(new Span(4, 7), words.get(1).span()); + } + + @Test + void testWithIsCopyOnAdd() { + final Document empty = Document.of("42"); + final Document grown = empty.with(NUMBERS, + List.of(new Annotation<>(new Span(0, 2), 42))); + assertTrue(empty.layers().isEmpty()); + assertEquals(Set.of(NUMBERS), grown.layers()); + // unchanged layers are shared, not copied + final Document both = grown.with(WORDS, List.of()); + assertSame(grown.get(NUMBERS), both.get(NUMBERS)); + } + + @Test + void testEqualKeysFromDifferentConstantsInteroperate() { + final Document document = Document.of("the") + .with(LayerKey.of("words", String.class), + List.of(new Annotation<>(new Span(0, 3), "the"))); + assertEquals(1, document.get(WORDS).size()); + assertNotEquals(WORDS, LayerKey.of("words", CharSequence.class)); + } + + @Test + void testDuplicateLayerThrows() { + final Document document = Document.of("the").with(WORDS, List.of()); + assertThrows(IllegalArgumentException.class, () -> document.with(WORDS, List.of())); + } + + @Test + void testSpanBeyondTextThrows() { + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(WORDS, List.of(new Annotation<>(new Span(0, 4), "the?")))); + } + + @Test + void testValueTypeIsCheckedOnInsertion() { + // a raw-typed caller cannot smuggle a mismatched value past the layer type + @SuppressWarnings({"unchecked", "rawtypes"}) + final LayerKey raw = (LayerKey) NUMBERS; + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(raw, List.of(new Annotation<>(new Span(0, 3), "not a number")))); + } + + @Test + void testNullArgumentsThrow() { + final Document document = Document.of("the"); + assertThrows(IllegalArgumentException.class, () -> Document.of(null)); + assertThrows(IllegalArgumentException.class, () -> document.get(null)); + assertThrows(IllegalArgumentException.class, () -> document.with(null, List.of())); + assertThrows(IllegalArgumentException.class, () -> document.with(WORDS, null)); + } + + @Test + void testAnnotationValidation() { + assertThrows(IllegalArgumentException.class, () -> new Annotation<>(null, "the")); + assertThrows(IllegalArgumentException.class, () -> new Annotation<>(new Span(0, 3), null)); + } + + @Test + void testLayerKeyValidation() { + assertThrows(IllegalArgumentException.class, () -> LayerKey.of(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.of(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.of("words", null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java new file mode 100644 index 0000000000..38478ac900 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests that {@link NameFinderAnnotator} maps token-index mentions to character spans on + * the original text and clears the finder's adaptive data per document. + */ +public class NameFinderAnnotatorTest { + + @Test + void testTokenIndexSpansBecomeCharacterSpans() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + // "New York" as a two-token person-free location mention + return new Span[] {new Span(1, 3, "location")}; + } + + @Override + public void clearAdaptiveData() { + cleared.incrementAndGet(); + } + }; + + final Document document = Document.of("in New York today") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "in"), + new Annotation<>(new Span(3, 6), "New"), + new Annotation<>(new Span(7, 11), "York"), + new Annotation<>(new Span(12, 17), "today"))); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + final List> entities = annotated.get(Layers.ENTITIES); + assertEquals(1, entities.size()); + assertEquals(new Span(3, 11, "location"), entities.get(0).span()); + assertEquals("location", entities.get(0).value()); + assertEquals("New York", + entities.get(0).span().getCoveredText(annotated.text()).toString()); + assertEquals(1, cleared.get()); + } + + @Test + void testMissingTokenLayerThrows() { + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(Document.of("no tokens"))); + } +} From 3e382b31b9d2ce73bbe16b2503c423fc86e3352e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 09:00:07 -0400 Subject: [PATCH 02/92] OPENNLP-1888: Lemma and stem layer adapters --- .../tools/lemmatizer/LemmatizerAnnotator.java | 94 +++++++++++++++++++ .../tools/stemmer/StemmerAnnotator.java | 81 ++++++++++++++++ .../lemmatizer/LemmatizerAnnotatorTest.java | 82 ++++++++++++++++ .../tools/stemmer/StemmerAnnotatorTest.java | 56 +++++++++++ 4 files changed, 313 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java new file mode 100644 index 0000000000..f1733da4eb --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link Lemmatizer} to the document pipeline: lemmatizes the token layer with + * its tags and provides {@link #LEMMAS}, one annotation per token on the token's span. + * + * @since 3.0.0 + */ +public class LemmatizerAnnotator implements DocumentAnnotator { + + /** Lemmas; aligned with the token layer, each annotation on its token's span. */ + public static final LayerKey LEMMAS = LayerKey.of("lemmas", String.class); + + private final Lemmatizer lemmatizer; + + /** + * Initializes the adapter. + * + * @param lemmatizer The lemmatizer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code lemmatizer} is {@code null}. + */ + public LemmatizerAnnotator(Lemmatizer lemmatizer) { + if (lemmatizer == null) { + throw new IllegalArgumentException("lemmatizer must not be null"); + } + this.lemmatizer = lemmatizer; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final String[] words = new String[tokens.size()]; + final String[] posTags = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + posTags[i] = tags.get(i).value(); + } + final String[] lemmas = lemmatizer.lemmatize(words, posTags); + if (lemmas.length != words.length) { + throw new IllegalArgumentException( + "lemmatizer returned " + lemmas.length + " lemmas for " + words.length + " tokens"); + } + final List> layer = new ArrayList<>(lemmas.length); + for (int i = 0; i < lemmas.length; i++) { + layer.add(new Annotation<>(tokens.get(i).span(), lemmas[i])); + } + return document.with(LEMMAS, layer); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS, Layers.POS_TAGS); + } + + @Override + public Set> provides() { + return Set.of(LEMMAS); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java new file mode 100644 index 0000000000..da3b109a66 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link Stemmer} to the document pipeline: stems the token layer and provides + * {@link #STEMS}, one annotation per token on the token's span. + * + *

Stemming needs no tags, so unlike lemmatization this annotator only requires the + * token layer.

+ * + * @since 3.0.0 + */ +public class StemmerAnnotator implements DocumentAnnotator { + + /** Stems; aligned with the token layer, each annotation on its token's span. */ + public static final LayerKey STEMS = LayerKey.of("stems", String.class); + + private final Stemmer stemmer; + + /** + * Initializes the adapter. + * + * @param stemmer The stemmer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code stemmer} is {@code null}. + */ + public StemmerAnnotator(Stemmer stemmer) { + if (stemmer == null) { + throw new IllegalArgumentException("stemmer must not be null"); + } + this.stemmer = stemmer; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + final List> layer = new ArrayList<>(tokens.size()); + for (final Annotation token : tokens) { + layer.add(new Annotation<>(token.span(), stemmer.stem(token.value()).toString())); + } + return document.with(STEMS, layer); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(STEMS); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java new file mode 100644 index 0000000000..6b2fb8b8d0 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +public class LemmatizerAnnotatorTest { + + /** Lowercases verbs and keeps everything else, enough to observe the adapter. */ + private static final Lemmatizer FIXTURE = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + final String[] lemmas = new String[toks.length]; + for (int i = 0; i < toks.length; i++) { + lemmas[i] = "VERB".equals(tags[i]) ? "run" : toks[i]; + } + return lemmas; + } + + @Override + public List> lemmatize(List toks, List tags) { + throw new UnsupportedOperationException(); + } + }; + + @Test + void testLemmasAlignWithTokens() { + final Document document = Document.of("She ran home") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "She"), + new Annotation<>(new Span(4, 7), "ran"), + new Annotation<>(new Span(8, 12), "home"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PRON"), + new Annotation<>(new Span(4, 7), "VERB"), + new Annotation<>(new Span(8, 12), "NOUN"))); + + final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); + + final List> lemmas = lemmatized.get(LemmatizerAnnotator.LEMMAS); + Assertions.assertEquals(3, lemmas.size()); + Assertions.assertEquals("run", lemmas.get(1).value()); + Assertions.assertEquals(new Span(4, 7), lemmas.get(1).span()); + Assertions.assertEquals("home", lemmas.get(2).value()); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(null)); + final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); + Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + final Document misaligned = Document.of("a b") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))) + .with(Layers.POS_TAGS, List.of()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(misaligned)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java new file mode 100644 index 0000000000..931b320e39 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +public class StemmerAnnotatorTest { + + @Test + void testStemsAlignWithTokens() { + final Document document = Document.of("running dogs") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 7), "running"), + new Annotation<>(new Span(8, 12), "dogs"))); + + final Document stemmed = new StemmerAnnotator( + new PorterStemmer()).annotate(document); + + final List> stems = stemmed.get(StemmerAnnotator.STEMS); + Assertions.assertEquals(2, stems.size()); + Assertions.assertEquals("run", stems.get(0).value()); + Assertions.assertEquals(new Span(0, 7), stems.get(0).span()); + Assertions.assertEquals("dog", stems.get(1).value()); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new StemmerAnnotator(null)); + final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); + Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + } +} From e8a2ca47f8dc465e6f4db76c35c6fe765473d77c Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:29:09 -0400 Subject: [PATCH 03/92] OPENNLP-1888: Pipeline example and contract tests for the document container, javadoc precision pass --- .../tools/document/ImmutableDocument.java | 3 +- .../java/opennlp/tools/document/Layers.java | 2 +- .../tools/document/DocumentContractTest.java | 281 +++++++++++++++++ .../document/DocumentPipelineExampleTest.java | 290 ++++++++++++++++++ .../tools/lemmatizer/LemmatizerAnnotator.java | 5 +- .../tools/stemmer/StemmerAnnotator.java | 9 +- 6 files changed, 584 insertions(+), 6 deletions(-) create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index 2ec407ee3e..43276b4cef 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -69,7 +69,8 @@ public List> get(LayerKey layer) { if (annotations == null) { return List.of(); } - // safe: with(LayerKey, List) verified every value against the key's type on insertion + // This cast is safe because with(LayerKey, List) verified every value against the + // key's type when the layer was inserted. return (List>) (List) annotations; } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index 9ff47cd2c3..99f0730550 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -50,6 +50,6 @@ public final class Layers { public static final LayerKey ENTITIES = LayerKey.of("entities", String.class); private Layers() { - // constants only + // This class holds constants only and is never instantiated. } } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java new file mode 100644 index 0000000000..c180b1bf9a --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins down the observable contract of the document container at its edges: key + * equality, layer ordering, span boundary cases, list immutability, the exact rejection + * messages, and the analyzer's build-time validation. Every expected value in this class + * is asserted exactly so any behavioral drift is caught, not just gross breakage. + */ +public class DocumentContractTest { + + private static final LayerKey WORDS = LayerKey.of("words", String.class); + + /** + * Verifies that two independently created keys with the same id and the same type are + * equal, hash alike, and therefore address the same layer, while remaining distinct + * instances. This is what lets separately compiled producers agree on a layer without + * sharing a constant. + */ + @Test + void testKeysWithSameIdAndTypeAddressTheSameLayer() { + final LayerKey first = LayerKey.of("words", String.class); + final LayerKey second = LayerKey.of("words", String.class); + assertNotSame(first, second); + assertEquals(first, second); + assertEquals(first.hashCode(), second.hashCode()); + assertEquals("words", first.toString()); + + final Document document = Document.of("the") + .with(first, List.of(new Annotation<>(new Span(0, 3), "the"))); + // Reading through the other, equal key yields the very layer added above. + assertEquals(1, document.get(second).size()); + assertEquals("the", document.get(second).get(0).value()); + // A duplicate add through the equal key is rejected like any duplicate. + assertThrows(IllegalArgumentException.class, () -> document.with(second, List.of())); + } + + /** + * Verifies that two keys sharing an id but differing in value type are unequal and + * denote two independent layers that can coexist on one document. + */ + @Test + void testKeysWithSameIdButDifferentTypesAreDifferentLayers() { + final LayerKey asString = LayerKey.of("marks", String.class); + final LayerKey asInteger = LayerKey.of("marks", Integer.class); + assertNotEquals(asString, asInteger); + + final Document document = Document.of("ab") + .with(asString, List.of(new Annotation<>(new Span(0, 1), "a"))) + .with(asInteger, List.of(new Annotation<>(new Span(1, 2), 7))); + assertEquals(Set.of(asString, asInteger), document.layers()); + assertEquals("a", document.get(asString).get(0).value()); + assertEquals(7, document.get(asInteger).get(0).value()); + } + + /** + * Verifies that a layer preserves the insertion order of its annotations: the + * container does not sort by span, so a producer that wants span order must supply + * span order. + */ + @Test + void testLayerPreservesInsertionOrder() { + final Document document = Document.of("the dog") + .with(WORDS, List.of( + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(0, 3), "the"))); + final List> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals(new Span(4, 7), words.get(0).span()); + assertEquals("dog", words.get(0).value()); + assertEquals(new Span(0, 3), words.get(1).span()); + assertEquals("the", words.get(1).value()); + } + + /** + * Verifies that several annotations may share one span within a layer, for example + * alternative readings of the same region, and all of them are retained in order. + */ + @Test + void testAnnotationsWithIdenticalSpansAreAllRetained() { + final Document document = Document.of("bank") + .with(WORDS, List.of( + new Annotation<>(new Span(0, 4), "institution"), + new Annotation<>(new Span(0, 4), "riverside"))); + final List> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals("institution", words.get(0).value()); + assertEquals("riverside", words.get(1).value()); + assertEquals(words.get(0).span(), words.get(1).span()); + } + + /** + * Verifies that zero-length spans are accepted anywhere within the bounds, including + * at the very end of the text where start and end equal the text length. + */ + @Test + void testZeroLengthSpansAreAccepted() { + final Document document = Document.of("ab") + .with(WORDS, List.of( + new Annotation<>(new Span(1, 1), "between"), + new Annotation<>(new Span(2, 2), "at the end"))); + final List> words = document.get(WORDS); + assertEquals(new Span(1, 1), words.get(0).span()); + assertEquals(new Span(2, 2), words.get(1).span()); + assertEquals(0, words.get(0).span().length()); + } + + /** + * Verifies that a span reaching past the end of the text is rejected on insertion + * with a message naming the span, the text length, and the layer. + */ + @Test + void testSpanBeyondTextLengthIsRejectedWithExactMessage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(WORDS, + List.of(new Annotation<>(new Span(0, 4), "the?")))); + assertEquals("span [0..4) exceeds the text length 3 in layer words", + e.getMessage()); + } + + /** + * Verifies that adding a layer under a key that is already present is rejected with a + * message naming the offending layer. + */ + @Test + void testDuplicateLayerIsRejectedWithExactMessage() { + final Document document = Document.of("the").with(WORDS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> document.with(WORDS, List.of())); + assertEquals("layer is already present: words", e.getMessage()); + } + + /** + * Verifies that reading an absent layer yields an empty, unmodifiable list rather + * than {@code null}, so callers can iterate without a presence check. + */ + @Test + void testAbsentLayerReadsAsUnmodifiableEmptyList() { + final List> absent = Document.of("the").get(WORDS); + assertTrue(absent.isEmpty()); + assertThrows(UnsupportedOperationException.class, + () -> absent.add(new Annotation<>(new Span(0, 3), "the"))); + } + + /** + * Verifies that the list returned for a present layer is unmodifiable and detached + * from the caller's input list: mutating the input after the add does not change the + * document. + */ + @Test + void testPresentLayerListIsUnmodifiableAndDetachedFromInput() { + final List> input = new ArrayList<>(); + input.add(new Annotation<>(new Span(0, 3), "the")); + final Document document = Document.of("the").with(WORDS, input); + + final List> words = document.get(WORDS); + assertThrows(UnsupportedOperationException.class, () -> words.remove(0)); + assertThrows(UnsupportedOperationException.class, + () -> words.add(new Annotation<>(new Span(0, 3), "the"))); + + input.clear(); + assertEquals(1, document.get(WORDS).size()); + } + + /** + * Verifies that the layer key set exposed by a document cannot be mutated by callers. + */ + @Test + void testLayerKeySetIsUnmodifiable() { + final Document document = Document.of("the").with(WORDS, List.of()); + assertThrows(UnsupportedOperationException.class, () -> document.layers().clear()); + } + + /** + * Verifies that an analyzer whose annotator requires a layer no earlier annotator + * provides fails at build time with a message naming the annotator and the missing + * layer. The annotator under test overrides {@code toString()} so the whole message + * can be asserted exactly. + */ + @Test + void testUnsatisfiedRequirementFailsAtBuildTimeWithExactMessage() { + final DocumentAnnotator needsTags = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + throw new IllegalStateException("must never run; the pipeline must not build"); + } + + @Override + public Set> requires() { + return Set.of(Layers.POS_TAGS); + } + + @Override + public Set> provides() { + return Set.of(WORDS); + } + + @Override + public String toString() { + return "tag-consumer"; + } + }; + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder().add(needsTags); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, builder::build); + assertEquals("annotator tag-consumer requires layer pos," + + " which no earlier annotator provides", e.getMessage()); + } + + /** + * Verifies that building an analyzer without any annotator fails with a message + * stating that a pipeline needs at least one annotator. + */ + @Test + void testEmptyPipelineFailsWithExactMessage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnalyzer.builder().build()); + assertEquals("a pipeline needs at least one annotator", e.getMessage()); + } + + /** + * Verifies that the value type travels through {@link LayerKey}: a layer added under + * an {@code Integer} key reads back as {@code Annotation}, so its values + * participate in arithmetic without a cast, and a mismatched value can never enter + * the layer in the first place. + */ + @Test + void testValueTypeTravelsThroughTheKey() { + final LayerKey counts = LayerKey.of("counts", Integer.class); + final Document document = Document.of("ab cd") + .with(counts, List.of( + new Annotation<>(new Span(0, 2), 2), + new Annotation<>(new Span(3, 5), 40))); + int sum = 0; + for (final Annotation count : document.get(counts)) { + sum += count.value(); + } + assertEquals(42, sum); + + // The insertion-time check backs the typed read: a raw-typed caller cannot place a + // String under the Integer key. + @SuppressWarnings({"unchecked", "rawtypes"}) + final LayerKey raw = (LayerKey) LayerKey.of("counts2", Integer.class); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> Document.of("ab").with(raw, + List.of(new Annotation<>(new Span(0, 2), "not a number")))); + assertEquals("value of type java.lang.String does not match layer counts2", + e.getMessage()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java new file mode 100644 index 0000000000..31a2379380 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.postag.POSTagger; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Walks through the document pipeline the way a first-time user would: wrap existing + * analysis components in their adapter annotators, add one custom annotator, build a + * {@link DocumentAnalyzer}, analyze a two-sentence text, and read every layer back with + * its spans in original text coordinates. + * + *

The wrapped components are tiny deterministic stand-ins defined in this class, so + * every expected span and value below follows directly from the input text. The point + * under demonstration is how the layers connect, not the quality of any single step.

+ */ +public class DocumentPipelineExampleTest { + + /** + * The key of the custom layer produced by {@link TokenLengthAnnotator}. Any producer + * may introduce such a key in its own code; the container needs no change for it. + */ + private static final LayerKey TOKEN_LENGTHS = + LayerKey.of("token-lengths", Integer.class); + + /** + * A deterministic sentence detector that ends a sentence after every period and + * expects a single space between sentences. Only the span-producing method is + * implemented because the adapter calls no other method. + */ + private static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the adapter only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A deterministic tokenizer that splits on single space characters and keeps all + * other characters, including sentence-final periods, attached to their token. Only + * the span-producing method is implemented because the adapter calls no other method. + */ + private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the adapter only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A deterministic tagger backed by a fixed dictionary covering exactly the tokens of + * the example text. An unknown token fails the test immediately rather than receiving + * a silent fallback tag. + */ + private static final POSTagger DICTIONARY_TAGGER = new POSTagger() { + + private final Map tagsByToken = Map.of( + "The", "DT", "dog", "NN", "barks.", "VBZ", "It", "PRP", "naps.", "VBZ"); + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + final String tag = tagsByToken.get(sentence[i]); + if (tag == null) { + throw new IllegalArgumentException("no tag defined for token: " + sentence[i]); + } + tags[i] = tag; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + }; + + /** + * A custom pipeline step written directly against {@link DocumentAnnotator}: it reads + * the token layer and provides {@link #TOKEN_LENGTHS}, one annotation per token on the + * token's span, whose value is the character length of the token text. + */ + private static final class TokenLengthAnnotator implements DocumentAnnotator { + + /** + * Adds the {@link #TOKEN_LENGTHS} layer computed from {@link Layers#TOKENS}. + * + * @param document The document to annotate. Must not be {@code null} and must + * contain the token layer. + * @return A new {@link Document} carrying the token length layer. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or the + * token layer is absent. + */ + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final List> lengths = new ArrayList<>(tokens.size()); + for (final Annotation token : tokens) { + lengths.add(new Annotation<>(token.span(), token.value().length())); + } + return document.with(TOKEN_LENGTHS, lengths); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(TOKEN_LENGTHS); + } + } + + /** + * Runs the full pipeline story: sentences, tokens, part-of-speech tags, and one custom + * layer over a two-sentence text, then verifies every annotation of every layer, span + * by span, in original text coordinates. + */ + @Test + void testFullPipelineStory() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new POSTaggerAnnotator(DICTIONARY_TAGGER)) + .add(new TokenLengthAnnotator()) + .build(); + + final Document document = analyzer.analyze("The dog barks. It naps."); + + // The document carries exactly the four layers the pipeline provides. + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, TOKEN_LENGTHS), + document.layers()); + assertEquals("The dog barks. It naps.", document.text()); + + // Sentence layer: one annotation per sentence, covering it in document coordinates. + final List> sentences = document.get(Layers.SENTENCES); + assertEquals(2, sentences.size()); + assertEquals(new Span(0, 14), sentences.get(0).span()); + assertEquals("The dog barks.", sentences.get(0).value()); + assertEquals(new Span(15, 23), sentences.get(1).span()); + assertEquals("It naps.", sentences.get(1).value()); + + // Token layer: five tokens; the second sentence's spans are shifted back to + // document coordinates, so every span can index into the original text. + final List> tokens = document.get(Layers.TOKENS); + assertEquals(5, tokens.size()); + assertEquals(new Span(0, 3), tokens.get(0).span()); + assertEquals("The", tokens.get(0).value()); + assertEquals(new Span(4, 7), tokens.get(1).span()); + assertEquals("dog", tokens.get(1).value()); + assertEquals(new Span(8, 14), tokens.get(2).span()); + assertEquals("barks.", tokens.get(2).value()); + assertEquals(new Span(15, 17), tokens.get(3).span()); + assertEquals("It", tokens.get(3).value()); + assertEquals(new Span(18, 23), tokens.get(4).span()); + assertEquals("naps.", tokens.get(4).value()); + + // Tag layer: aligned with the token layer by position, each tag on its token's span. + final List> tags = document.get(Layers.POS_TAGS); + assertEquals(5, tags.size()); + assertEquals("DT", tags.get(0).value()); + assertEquals("NN", tags.get(1).value()); + assertEquals("VBZ", tags.get(2).value()); + assertEquals("PRP", tags.get(3).value()); + assertEquals("VBZ", tags.get(4).value()); + for (int i = 0; i < tags.size(); i++) { + assertEquals(tokens.get(i).span(), tags.get(i).span()); + } + + // Custom layer: the container returns it as List>, so the + // values are used as numbers without a cast. + final List> lengths = document.get(TOKEN_LENGTHS); + assertEquals(5, lengths.size()); + assertEquals(3, lengths.get(0).value()); + assertEquals(3, lengths.get(1).value()); + assertEquals(6, lengths.get(2).value()); + assertEquals(2, lengths.get(3).value()); + assertEquals(5, lengths.get(4).value()); + for (int i = 0; i < lengths.size(); i++) { + assertEquals(tokens.get(i).span(), lengths.get(i).span()); + } + + // Every span refers to the original text, so covered text always round-trips. + for (final Annotation token : tokens) { + assertEquals(token.value(), + token.span().getCoveredText(document.text()).toString()); + } + } + + /** + * Verifies that the analyzer leaves the input untouched between calls: analyzing two + * texts with the same analyzer yields two independent documents. + */ + @Test + void testAnalyzerIsReusableAcrossTexts() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .build(); + + final Document first = analyzer.analyze("The dog barks."); + final Document second = analyzer.analyze("It naps."); + + assertEquals(3, first.get(Layers.TOKENS).size()); + assertEquals(2, second.get(Layers.TOKENS).size()); + assertEquals("The dog barks.", first.text()); + assertEquals("It naps.", second.text()); + assertEquals(1, first.get(Layers.SENTENCES).size()); + assertEquals(1, second.get(Layers.SENTENCES).size()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index f1733da4eb..37fc3478ad 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -35,7 +35,10 @@ */ public class LemmatizerAnnotator implements DocumentAnnotator { - /** Lemmas; aligned with the token layer, each annotation on its token's span. */ + /** + * The lemma layer. It is aligned with the token layer by position, and each annotation + * carries the lemma of its token on that token's span. + */ public static final LayerKey LEMMAS = LayerKey.of("lemmas", String.class); private final Lemmatizer lemmatizer; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index da3b109a66..8d76d018ac 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -31,14 +31,17 @@ * Adapts a {@link Stemmer} to the document pipeline: stems the token layer and provides * {@link #STEMS}, one annotation per token on the token's span. * - *

Stemming needs no tags, so unlike lemmatization this annotator only requires the - * token layer.

+ *

Stemming operates on the token surface alone, so this annotator requires only the + * token layer; no part-of-speech tags are involved.

* * @since 3.0.0 */ public class StemmerAnnotator implements DocumentAnnotator { - /** Stems; aligned with the token layer, each annotation on its token's span. */ + /** + * The stem layer. It is aligned with the token layer by position, and each annotation + * carries the stem of its token on that token's span. + */ public static final LayerKey STEMS = LayerKey.of("stems", String.class); private final Stemmer stemmer; From e9b53a7a64879b4a6549f1c6d6054da0c36ae100 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 21:14:22 -0400 Subject: [PATCH 04/92] OPENNLP-1888: Parse per sentence in the adapters, distinguish empty from missing layers, validate providers at build time --- .../tools/document/DocumentAnalyzer.java | 31 ++- .../tools/document/DocumentAnnotator.java | 4 + .../tools/document/NameFinderAnnotator.java | 97 +++++++-- .../tools/document/POSTaggerAnnotator.java | 74 +++++-- .../tools/document/TokenizerAnnotator.java | 12 +- .../tools/document/DocumentAnalyzerTest.java | 49 +++++ .../tools/document/DocumentContractTest.java | 27 +++ .../document/NameFinderAnnotatorTest.java | 130 +++++++++++ .../document/POSTaggerAnnotatorTest.java | 204 ++++++++++++++++++ .../tools/lemmatizer/LemmatizerAnnotator.java | 23 ++ .../tools/stemmer/StemmerAnnotator.java | 17 ++ .../lemmatizer/LemmatizerAnnotatorTest.java | 33 +++ .../tools/stemmer/StemmerAnnotatorTest.java | 24 +++ 13 files changed, 679 insertions(+), 46 deletions(-) create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java index 4ca353ca1c..e12a468b5a 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java @@ -18,18 +18,19 @@ package opennlp.tools.document; import java.util.ArrayList; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; -import java.util.Set; +import java.util.Map; /** * Runs a fixed sequence of {@link DocumentAnnotator annotators} over a text, producing * one {@link Document} that carries every step's layers. * *

The pipeline is validated at build time: every annotator's required layers must be - * provided by an earlier annotator, so a misordered pipeline fails when it is assembled - * rather than midway through a document. The analyzer holds no per-call state; it is as - * thread-safe as the annotators it is built from.

+ * provided by an earlier annotator, and no two annotators may provide the same layer, so + * a misordered or conflicting pipeline fails when it is assembled rather than midway + * through a document. The analyzer holds no per-call state; it is as thread-safe as the + * annotators it is built from.

* * @since 3.0.0 */ @@ -93,22 +94,30 @@ public Builder add(DocumentAnnotator annotator) { * Validates the pipeline and builds the analyzer. * * @return A {@link DocumentAnalyzer}. Never {@code null}. - * @throws IllegalArgumentException Thrown if the pipeline is empty or an annotator - * requires a layer no earlier annotator provides. + * @throws IllegalArgumentException Thrown if the pipeline is empty, an annotator + * requires a layer no earlier annotator provides, or two annotators provide + * the same layer. */ public DocumentAnalyzer build() { if (annotators.isEmpty()) { throw new IllegalArgumentException("a pipeline needs at least one annotator"); } - final Set> available = new HashSet<>(); - for (final DocumentAnnotator annotator : annotators) { + final Map, Integer> providers = new HashMap<>(); + for (int position = 0; position < annotators.size(); position++) { + final DocumentAnnotator annotator = annotators.get(position); for (final LayerKey required : annotator.requires()) { - if (!available.contains(required)) { + if (!providers.containsKey(required)) { throw new IllegalArgumentException("annotator " + annotator + " requires layer " + required + ", which no earlier annotator provides"); } } - available.addAll(annotator.provides()); + for (final LayerKey provided : annotator.provides()) { + final Integer earlier = providers.putIfAbsent(provided, position); + if (earlier != null) { + throw new IllegalArgumentException("annotators at positions " + earlier + + " and " + position + " both provide layer " + provided); + } + } } return new DocumentAnalyzer(List.copyOf(annotators)); } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java index 9ea2c65a4d..3f44a6d856 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java @@ -36,6 +36,10 @@ public interface DocumentAnnotator { /** * Annotates a document. * + *

A required layer must be present on the document, but it may be empty: an empty + * required layer is valid input and yields the annotator's provided layers present but + * empty, so a pipeline degrades gracefully on documents without content.

+ * * @param document The document to annotate. Must not be {@code null} and must contain * every layer named by {@link #requires()}. * @return A new {@link Document} carrying the layers named by {@link #provides()} in diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java index f5bfecdaf8..3902d41867 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -25,17 +25,32 @@ import opennlp.tools.util.Span; /** - * Adapts a {@link TokenNameFinder} to the document pipeline: reads {@link Layers#TOKENS}, - * maps the finder's token-index spans to character spans on the original text, and - * provides {@link Layers#ENTITIES} carrying the entity type. + * Adapts a {@link TokenNameFinder} to the document pipeline: reads + * {@link Layers#SENTENCES} and {@link Layers#TOKENS}, maps the finder's token-index + * spans to character spans on the original text, and provides {@link Layers#ENTITIES} + * carrying the entity type. * - *

The finder's adaptive data is cleared after each document, so document order does - * not leak between pipeline runs.

+ *

Each sentence's tokens are passed to {@link TokenNameFinder#find(String[])} as one + * sequence, the way the finder contract expects its input, so no mention can straddle a + * sentence boundary. The finder's adaptive data is cleared once after all sentences of a + * document are processed, as the {@link TokenNameFinder#clearAdaptiveData()} contract + * asks, so document order does not leak between pipeline runs.

+ * + *

Spans the finder returns without a type are recorded with the {@link #UNTYPED} + * entity type.

* * @since 3.0.0 */ public class NameFinderAnnotator implements DocumentAnnotator { + /** + * The entity type recorded when the wrapped finder returns a span without a type. It + * equals {@link opennlp.tools.namefind.NameSample#DEFAULT_TYPE}. Type-aware consumers + * should treat this label as an unknown type rather than as a distinct one, since it + * carries no information about what kind of entity was found. + */ + public static final String UNTYPED = "default"; + private final TokenNameFinder finder; /** @@ -51,26 +66,74 @@ public NameFinderAnnotator(TokenNameFinder finder) { this.finder = finder; } + /** + * Finds names sentence by sentence and adds the {@link Layers#ENTITIES} layer. + * + *

For every sentence, the tokens whose spans lie inside the sentence span are + * passed to the finder as one sequence, and each sentence-local mention is mapped + * through the sentence's first token position onto character spans of the original + * text. The required layers must be present, but they may be empty: a document + * without sentences or tokens yields a present-but-empty entity layer, and a sentence + * containing no tokens contributes nothing. A mention without a type is recorded with + * the type {@link #UNTYPED}.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link Layers#ENTITIES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * sentence layer or the token layer is absent, or a token lies outside every + * sentence. + */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } - final List> tokens = document.get(Layers.TOKENS); - if (tokens.isEmpty()) { + final Set> present = document.layers(); + if (!present.contains(Layers.SENTENCES)) { throw new IllegalArgumentException("document lacks the required layer " - + Layers.TOKENS); + + Layers.SENTENCES); } - final String[] words = new String[tokens.size()]; - for (int i = 0; i < words.length; i++) { - words[i] = tokens.get(i).value(); + if (!present.contains(Layers.TOKENS)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); } + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); final List> entities = new ArrayList<>(); - for (final Span mention : finder.find(words)) { - final int start = tokens.get(mention.getStart()).span().getStart(); - final int end = tokens.get(mention.getEnd() - 1).span().getEnd(); - final String type = mention.getType() == null ? "default" : mention.getType(); - entities.add(new Annotation<>(new Span(start, end, type), type)); + // Walk the token layer once: both layers are in text order, so each sentence + // consumes the contiguous run of tokens whose spans it encloses. + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + } + // The finder indexes within the sentence; shifting by the sentence's first token + // position turns every mention boundary into a document-wide token index, whose + // token spans already refer to the original text. + for (final Span mention : finder.find(words)) { + final int start = tokens.get(first + mention.getStart()).span().getStart(); + final int end = tokens.get(first + mention.getEnd() - 1).span().getEnd(); + final String type = mention.getType() == null ? UNTYPED : mention.getType(); + entities.add(new Annotation<>(new Span(start, end, type), type)); + } + } + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); } finder.clearAdaptiveData(); return document.with(Layers.ENTITIES, entities); @@ -78,7 +141,7 @@ public Document annotate(Document document) { @Override public Set> requires() { - return Set.of(Layers.TOKENS); + return Set.of(Layers.SENTENCES, Layers.TOKENS); } @Override diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java index 1234b6803d..478f210b47 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -24,8 +24,15 @@ import opennlp.tools.postag.POSTagger; /** - * Adapts a {@link POSTagger} to the document pipeline: reads {@link Layers#TOKENS} and - * provides {@link Layers#POS_TAGS}, one tag annotation per token on the token's span. + * Adapts a {@link POSTagger} to the document pipeline: reads {@link Layers#SENTENCES} + * and {@link Layers#TOKENS} and provides {@link Layers#POS_TAGS}, one tag annotation per + * token on the token's span. + * + *

Each sentence is tagged separately, the way the tagger contract expects its input, + * so tagging decisions never cross a sentence boundary. Token spans already refer to the + * original document text, so only the token sequence handed to the tagger is sliced per + * sentence; the produced tag layer stays aligned with {@link Layers#TOKENS} by + * position.

* * @since 3.0.0 */ @@ -46,31 +53,74 @@ public POSTaggerAnnotator(POSTagger tagger) { this.tagger = tagger; } + /** + * Tags the document sentence by sentence and adds the {@link Layers#POS_TAGS} layer. + * + *

For every sentence, the tokens whose spans lie inside the sentence span are + * tagged as one sequence, and each tag is emitted on its token's span. The required + * layers must be present, but they may be empty: a document without sentences or + * tokens yields a present-but-empty tag layer, and a sentence containing no tokens + * contributes nothing.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link Layers#POS_TAGS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * sentence layer or the token layer is absent, or a token lies outside every + * sentence. + */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } - final List> tokens = document.get(Layers.TOKENS); - if (tokens.isEmpty()) { + final Set> present = document.layers(); + if (!present.contains(Layers.SENTENCES)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.SENTENCES); + } + if (!present.contains(Layers.TOKENS)) { throw new IllegalArgumentException("document lacks the required layer " + Layers.TOKENS); } - final String[] words = new String[tokens.size()]; - for (int i = 0; i < words.length; i++) { - words[i] = tokens.get(i).value(); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> tagAnnotations = new ArrayList<>(tokens.size()); + // Walk the token layer once: both layers are in text order, so each sentence + // consumes the contiguous run of tokens whose spans it encloses. + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + } + final String[] tags = tagger.tag(words); + for (int i = 0; i < count; i++) { + tagAnnotations.add(new Annotation<>(tokens.get(first + i).span(), tags[i])); + } } - final String[] tags = tagger.tag(words); - final List> tagAnnotations = new ArrayList<>(tags.length); - for (int i = 0; i < tags.length; i++) { - tagAnnotations.add(new Annotation<>(tokens.get(i).span(), tags[i])); + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); } return document.with(Layers.POS_TAGS, tagAnnotations); } @Override public Set> requires() { - return Set.of(Layers.TOKENS); + return Set.of(Layers.SENTENCES, Layers.TOKENS); } @Override diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java index 625410a181..5c21111a65 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -28,9 +28,10 @@ * Adapts a {@link Tokenizer} to the document pipeline: provides {@link Layers#TOKENS}. * *

When {@link Layers#SENTENCES} is present, each sentence is tokenized separately and - * the token spans are shifted back to document coordinates; otherwise the whole text is - * tokenized at once. Either way, every token span refers to the original document - * text.

+ * the token spans are shifted back to document coordinates; a present-but-empty sentence + * layer therefore yields a present-but-empty token layer. Only when the sentence layer + * is absent is the whole text tokenized at once. Either way, every token span refers to + * the original document text.

* * @since 3.0.0 */ @@ -58,11 +59,10 @@ public Document annotate(Document document) { } final String text = document.text().toString(); final List> tokens = new ArrayList<>(); - final List> sentences = document.get(Layers.SENTENCES); - if (sentences.isEmpty()) { + if (!document.layers().contains(Layers.SENTENCES)) { addTokens(tokens, text, 0); } else { - for (final Annotation sentence : sentences) { + for (final Annotation sentence : document.get(Layers.SENTENCES)) { final Span span = sentence.span(); addTokens(tokens, text.substring(span.getStart(), span.getEnd()), span.getStart()); } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java index 534fe115c4..deb317640f 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; +import opennlp.tools.namefind.TokenNameFinder; import opennlp.tools.postag.POSTagger; import opennlp.tools.sentdetect.SentenceDetector; import opennlp.tools.tokenize.Tokenizer; @@ -29,6 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using simple @@ -138,6 +140,53 @@ void testPipelineProducesAlignedLayersInOriginalCoordinates() { assertEquals(tokens.get(2).span(), tags.get(2).span()); } + /** + * Verifies that a full pipeline over empty and whitespace-only input produces a + * document on which every provided layer is present and empty, rather than failing: + * zero sentences legitimately yield zero tokens, zero tags, and zero entities. + */ + @Test + void testEmptyAndBlankInputProduceEmptyLayers() { + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(SPLITTER)) + .add(new TokenizerAnnotator(WHITESPACE)) + .add(new POSTaggerAnnotator(TAGGER)) + .add(new NameFinderAnnotator(finder)) + .build(); + + for (final String text : new String[] {"", " "}) { + final Document document = analyzer.analyze(text); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, Layers.ENTITIES), + document.layers()); + for (final LayerKey layer : document.layers()) { + assertTrue(document.get(layer).isEmpty()); + } + } + } + + /** + * Verifies that a present-but-empty sentence layer is honored as "no sentences": the + * tokenizer adds a present-but-empty token layer instead of tokenizing the whole text. + */ + @Test + void testTokenizerHonorsPresentButEmptySentenceLayer() { + final Document document = new TokenizerAnnotator(WHITESPACE) + .annotate(Document.of("the dog").with(Layers.SENTENCES, List.of())); + assertTrue(document.layers().contains(Layers.TOKENS)); + assertTrue(document.get(Layers.TOKENS).isEmpty()); + } + @Test void testTokenizerWorksWithoutSentences() { final Document document = DocumentAnalyzer.builder() diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index c180b1bf9a..f46701ef12 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -238,6 +238,33 @@ public String toString() { + " which no earlier annotator provides", e.getMessage()); } + /** + * Verifies that an analyzer whose annotators would provide the same layer twice fails + * at build time with a message naming the layer and the positions of both providers, + * instead of crashing midway through the first document. + */ + @Test + void testDuplicateProviderFailsAtBuildTimeWithExactMessage() { + final DocumentAnnotator provider = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + throw new IllegalStateException("must never run; the pipeline must not build"); + } + + @Override + public Set> provides() { + return Set.of(WORDS); + } + }; + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(provider).add(provider); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, builder::build); + assertEquals("annotators at positions 0 and 1 both provide layer words", + e.getMessage()); + } + /** * Verifies that building an analyzer without any annotator fails with a message * stating that a pipeline needs at least one annotator. diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java index 38478ac900..971215d898 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -17,7 +17,9 @@ package opennlp.tools.document; +import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; @@ -27,6 +29,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests that {@link NameFinderAnnotator} maps token-index mentions to character spans on @@ -52,6 +55,8 @@ public void clearAdaptiveData() { }; final Document document = Document.of("in New York today") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 17), "in New York today"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 2), "in"), new Annotation<>(new Span(3, 6), "New"), @@ -84,4 +89,129 @@ public void clearAdaptiveData() { assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(finder).annotate(Document.of("no tokens"))); } + + /** + * Verifies that the finder is invoked once per sentence with exactly that sentence's + * tokens, that sentence-local mention indices are mapped through the sentence's first + * token position into document character spans, and that the adaptive data is cleared + * exactly once after the whole document. + */ + @Test + void testFindsPerSentenceAndMapsSentenceLocalIndices() { + final List> calls = new ArrayList<>(); + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + calls.add(List.of(tokens)); + // the first token of every sentence is a person mention, in sentence-local indices + return new Span[] {new Span(0, 1, "person")}; + } + + @Override + public void clearAdaptiveData() { + cleared.incrementAndGet(); + } + }; + + final Document document = Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + + assertEquals(List.of( + List.of("Ana", "runs."), + List.of("Bob", "sits.")), calls); + + final List> entities = annotated.get(Layers.ENTITIES); + assertEquals(2, entities.size()); + assertEquals(new Span(0, 3, "person"), entities.get(0).span()); + assertEquals(new Span(10, 13, "person"), entities.get(1).span()); + assertEquals("Bob", + entities.get(1).span().getCoveredText(annotated.text()).toString()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that the annotator declares both the sentence layer and the token layer as + * required, so a pipeline without a sentence step fails at build time. + */ + @Test + void testRequiresSentencesAndTokens() { + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), + new NameFinderAnnotator(finder).requires()); + } + + /** + * Verifies that present-but-empty sentence and token layers yield a present-but-empty + * entity layer without invoking the finder, rather than an exception. + */ + @Test + void testEmptyPresentLayersYieldEmptyEntityLayer() { + final AtomicInteger found = new AtomicInteger(); + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + found.incrementAndGet(); + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + final Document document = Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + + assertTrue(annotated.layers().contains(Layers.ENTITIES)); + assertTrue(annotated.get(Layers.ENTITIES).isEmpty()); + assertEquals(0, found.get()); + } + + /** + * Verifies that a document without a sentence layer is rejected with a message naming + * the missing layer. + */ + @Test + void testAbsentSentenceLayerThrowsWithExactMessage() { + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + final Document document = Document.of("Ana") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "Ana"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("document lacks the required layer sentences", e.getMessage()); + } } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java new file mode 100644 index 0000000000..95779fdd36 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.postag.POSTagger; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link POSTaggerAnnotator} tags one sentence per {@link POSTagger#tag(String[])} + * call, keeps the tag layer aligned with the token layer, distinguishes a present-but-empty + * required layer from an absent one, and rejects tokens outside every sentence. + */ +public class POSTaggerAnnotatorTest { + + /** + * A tagger that records the exact token sequence of every call and answers with one + * {@code "X"} tag per token, so the per-call slicing is observable. + */ + private static final class RecordingTagger implements POSTagger { + + private final List> calls = new ArrayList<>(); + + @Override + public String[] tag(String[] sentence) { + calls.add(List.of(sentence)); + final String[] tags = new String[sentence.length]; + Arrays.fill(tags, "X"); + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + } + + /** + * @return A two-sentence document with sentence and token layers over + * {@code "The dog barks. It naps."}. Never {@code null}. + */ + private static Document twoSentenceDocument() { + return Document.of("The dog barks. It naps.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 14), "The dog barks."), + new Annotation<>(new Span(15, 23), "It naps."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(8, 14), "barks."), + new Annotation<>(new Span(15, 17), "It"), + new Annotation<>(new Span(18, 23), "naps."))); + } + + /** + * Verifies that the tagger is invoked once per sentence with exactly that sentence's + * tokens, and that the resulting tag layer stays aligned with the token layer by + * position, each tag on its token's span. + */ + @Test + void testTagsEachSentenceSeparately() { + final RecordingTagger tagger = new RecordingTagger(); + final Document annotated = new POSTaggerAnnotator(tagger).annotate(twoSentenceDocument()); + + assertEquals(List.of( + List.of("The", "dog", "barks."), + List.of("It", "naps.")), tagger.calls); + + final List> tokens = annotated.get(Layers.TOKENS); + final List> tags = annotated.get(Layers.POS_TAGS); + assertEquals(tokens.size(), tags.size()); + for (int i = 0; i < tags.size(); i++) { + assertEquals(tokens.get(i).span(), tags.get(i).span()); + assertEquals("X", tags.get(i).value()); + } + } + + /** + * Verifies that the annotator declares both the sentence layer and the token layer as + * required, so a pipeline without a sentence step fails at build time. + */ + @Test + void testRequiresSentencesAndTokens() { + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), + new POSTaggerAnnotator(new RecordingTagger()).requires()); + } + + /** + * Verifies that present-but-empty sentence and token layers yield a present-but-empty + * tag layer without invoking the tagger, rather than an exception. + */ + @Test + void testEmptyPresentLayersYieldEmptyTagLayer() { + final RecordingTagger tagger = new RecordingTagger(); + final Document document = Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()); + + final Document annotated = new POSTaggerAnnotator(tagger).annotate(document); + + assertTrue(annotated.layers().contains(Layers.POS_TAGS)); + assertTrue(annotated.get(Layers.POS_TAGS).isEmpty()); + assertTrue(tagger.calls.isEmpty()); + } + + /** + * Verifies that a document without a sentence layer is rejected with a message naming + * the missing layer. + */ + @Test + void testAbsentSentenceLayerThrowsWithExactMessage() { + final Document document = Document.of("The dog") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); + assertEquals("document lacks the required layer sentences", e.getMessage()); + } + + /** + * Verifies that a document without a token layer is rejected with a message naming the + * missing layer. + */ + @Test + void testAbsentTokenLayerThrowsWithExactMessage() { + final Document document = Document.of("The dog") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 7), "The dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); + assertEquals("document lacks the required layer tokens", e.getMessage()); + } + + /** + * Verifies that a token whose span no sentence encloses is rejected with a message + * naming the token's span. + */ + @Test + void testTokenOutsideEverySentenceThrowsWithExactMessage() { + final Document document = Document.of("The dog") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "The"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); + assertEquals("token at [4..7) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies that a sentence containing no tokens contributes nothing: the tagger is + * never called with an empty sequence and the tag layer still matches the token layer. + */ + @Test + void testSentenceWithoutTokensContributesNothing() { + final RecordingTagger tagger = new RecordingTagger(); + final Document document = Document.of("The ???") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "???"))) + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "The"))); + + final Document annotated = new POSTaggerAnnotator(tagger).annotate(document); + + assertEquals(List.of(List.of("The")), tagger.calls); + assertEquals(1, annotated.get(Layers.POS_TAGS).size()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index 37fc3478ad..0561d667e4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -56,11 +56,34 @@ public LemmatizerAnnotator(Lemmatizer lemmatizer) { this.lemmatizer = lemmatizer; } + /** + * Lemmatizes the token layer with its tags and adds the {@link #LEMMAS} layer. + * + *

The required layers must be present, but they may be empty: a document without + * tokens yields a present-but-empty lemma layer.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#TOKENS} layer and a {@link Layers#POS_TAGS} layer + * of equal size. + * @return A new {@link Document} with the {@link #LEMMAS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * token layer or the tag layer is absent, the tag layer does not have exactly + * one tag per token, or the lemmatizer does not return one lemma per token. + */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } + if (!document.layers().contains(Layers.TOKENS)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + if (!document.layers().contains(Layers.POS_TAGS)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.POS_TAGS); + } final List> tokens = document.get(Layers.TOKENS); final List> tags = document.get(Layers.POS_TAGS); if (tags.size() != tokens.size()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index 8d76d018ac..330087f5d8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -59,11 +59,28 @@ public StemmerAnnotator(Stemmer stemmer) { this.stemmer = stemmer; } + /** + * Stems the token layer and adds the {@link #STEMS} layer. + * + *

The token layer must be present, but it may be empty: a document without tokens + * yields a present-but-empty stem layer.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#TOKENS} layer. + * @return A new {@link Document} with the {@link #STEMS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or the + * token layer is absent. + */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } + if (!document.layers().contains(Layers.TOKENS)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } final List> tokens = document.get(Layers.TOKENS); final List> layer = new ArrayList<>(tokens.size()); for (final Annotation token : tokens) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java index 6b2fb8b8d0..6a5596dfe8 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -79,4 +79,37 @@ void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(misaligned)); } + + /** + * Verifies that a document lacking a required layer is rejected with a message naming + * the missing layer, instead of silently producing an empty lemma layer. + */ + @Test + void testAbsentRequiredLayerThrowsWithExactMessage() { + final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); + final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no layers"))); + Assertions.assertEquals("document lacks the required layer tokens", e.getMessage()); + + final Document tokensOnly = Document.of("a") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))); + final IllegalArgumentException tagless = Assertions.assertThrows( + IllegalArgumentException.class, () -> annotator.annotate(tokensOnly)); + Assertions.assertEquals("document lacks the required layer pos", + tagless.getMessage()); + } + + /** + * Verifies that present-but-empty token and tag layers yield a present-but-empty lemma + * layer rather than an exception. + */ + @Test + void testEmptyPresentLayersYieldEmptyLemmaLayer() { + final Document document = Document.of("") + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of()); + final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); + Assertions.assertTrue(lemmatized.layers().contains(LemmatizerAnnotator.LEMMAS)); + Assertions.assertTrue(lemmatized.get(LemmatizerAnnotator.LEMMAS).isEmpty()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java index 931b320e39..86b3a82ec7 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -53,4 +53,28 @@ void testInvalidArguments() { final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); } + + /** + * Verifies that a document without a token layer is rejected with a message naming the + * missing layer, instead of silently producing an empty stem layer. + */ + @Test + void testAbsentTokenLayerThrowsWithExactMessage() { + final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); + final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no tokens"))); + Assertions.assertEquals("document lacks the required layer tokens", e.getMessage()); + } + + /** + * Verifies that a present-but-empty token layer yields a present-but-empty stem layer + * rather than an exception. + */ + @Test + void testEmptyPresentTokenLayerYieldsEmptyStemLayer() { + final Document document = Document.of("").with(Layers.TOKENS, List.of()); + final Document stemmed = new StemmerAnnotator(new PorterStemmer()).annotate(document); + Assertions.assertTrue(stemmed.layers().contains(StemmerAnnotator.STEMS)); + Assertions.assertTrue(stemmed.get(StemmerAnnotator.STEMS).isEmpty()); + } } From 6b384b670be9db2eae6022af725bd8f48c5b0f34 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 23:30:37 -0400 Subject: [PATCH 05/92] OPENNLP-1888: Lemmatize per sentence, validate adapter outputs, clear adaptive data on failure The lemmatizer adapter now slices tokens and tags per sentence like its POS and name-finder siblings, so lemmatization decisions never cross a sentence boundary, and it declares the sentence layer as required. The POS adapter rejects a tagger that returns a wrong tag count. The name-finder adapter rejects mentions whose token indices lie outside their sentence instead of silently reading the next sentence's tokens, clears adaptive data even when annotation fails, and derives UNTYPED from NameSample.DEFAULT_TYPE instead of re-declaring the literal. --- .../tools/document/NameFinderAnnotator.java | 87 +++++++----- .../tools/document/POSTaggerAnnotator.java | 8 +- .../document/NameFinderAnnotatorTest.java | 117 +++++++++++++++- .../document/POSTaggerAnnotatorTest.java | 38 +++++ .../tools/lemmatizer/LemmatizerAnnotator.java | 83 ++++++++--- .../lemmatizer/LemmatizerAnnotatorTest.java | 132 +++++++++++++++++- 6 files changed, 394 insertions(+), 71 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java index 3902d41867..b129b77b29 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Set; +import opennlp.tools.namefind.NameSample; import opennlp.tools.namefind.TokenNameFinder; import opennlp.tools.util.Span; @@ -32,9 +33,9 @@ * *

Each sentence's tokens are passed to {@link TokenNameFinder#find(String[])} as one * sequence, the way the finder contract expects its input, so no mention can straddle a - * sentence boundary. The finder's adaptive data is cleared once after all sentences of a - * document are processed, as the {@link TokenNameFinder#clearAdaptiveData()} contract - * asks, so document order does not leak between pipeline runs.

+ * sentence boundary. The finder's adaptive data is cleared exactly once per call, as the + * {@link TokenNameFinder#clearAdaptiveData()} contract asks, whether annotation succeeds + * or fails, so no document can leak finder state into the next one.

* *

Spans the finder returns without a type are recorded with the {@link #UNTYPED} * entity type.

@@ -45,11 +46,11 @@ public class NameFinderAnnotator implements DocumentAnnotator { /** * The entity type recorded when the wrapped finder returns a span without a type. It - * equals {@link opennlp.tools.namefind.NameSample#DEFAULT_TYPE}. Type-aware consumers - * should treat this label as an unknown type rather than as a distinct one, since it - * carries no information about what kind of entity was found. + * is {@link NameSample#DEFAULT_TYPE}. Type-aware consumers should treat this label as + * an unknown type rather than as a distinct one, since it carries no information about + * what kind of entity was found. */ - public static final String UNTYPED = "default"; + public static final String UNTYPED = NameSample.DEFAULT_TYPE; private final TokenNameFinder finder; @@ -83,8 +84,9 @@ public NameFinderAnnotator(TokenNameFinder finder) { * @return A new {@link Document} with the {@link Layers#ENTITIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * sentence layer or the token layer is absent, or a token lies outside every - * sentence. + * sentence layer or the token layer is absent, a token lies outside every + * sentence, or the finder returns a mention whose token indices lie outside + * its sentence's tokens. */ @Override public Document annotate(Document document) { @@ -104,38 +106,47 @@ public Document annotate(Document document) { final List> tokens = document.get(Layers.TOKENS); final List> entities = new ArrayList<>(); // Walk the token layer once: both layers are in text order, so each sentence - // consumes the contiguous run of tokens whose spans it encloses. - int next = 0; - for (final Annotation sentence : sentences) { - final int first = next; - while (next < tokens.size() - && tokens.get(next).span().getStart() >= sentence.span().getStart() - && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { - next++; + // consumes the contiguous run of tokens whose spans it encloses. The adaptive data + // is cleared even when annotation fails, so a rejected document cannot leak finder + // state into the next one. + try { + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + } + // The finder indexes within the sentence; shifting by the sentence's first token + // position turns every mention boundary into a document-wide token index, whose + // token spans already refer to the original text. + for (final Span mention : finder.find(words)) { + if (mention.getStart() < 0 || mention.getEnd() > count) { + throw new IllegalArgumentException("finder returned mention " + mention + + " outside the sentence's " + count + " tokens"); + } + final int start = tokens.get(first + mention.getStart()).span().getStart(); + final int end = tokens.get(first + mention.getEnd() - 1).span().getEnd(); + final String type = mention.getType() == null ? UNTYPED : mention.getType(); + entities.add(new Annotation<>(new Span(start, end, type), type)); + } } - final int count = next - first; - if (count == 0) { - continue; + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); } - final String[] words = new String[count]; - for (int i = 0; i < count; i++) { - words[i] = tokens.get(first + i).value(); - } - // The finder indexes within the sentence; shifting by the sentence's first token - // position turns every mention boundary into a document-wide token index, whose - // token spans already refer to the original text. - for (final Span mention : finder.find(words)) { - final int start = tokens.get(first + mention.getStart()).span().getStart(); - final int end = tokens.get(first + mention.getEnd() - 1).span().getEnd(); - final String type = mention.getType() == null ? UNTYPED : mention.getType(); - entities.add(new Annotation<>(new Span(start, end, type), type)); - } - } - if (next != tokens.size()) { - throw new IllegalArgumentException("token at " + tokens.get(next).span() - + " lies outside every sentence"); + } finally { + finder.clearAdaptiveData(); } - finder.clearAdaptiveData(); return document.with(Layers.ENTITIES, entities); } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java index 478f210b47..f7564cd648 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -68,8 +68,8 @@ public POSTaggerAnnotator(POSTagger tagger) { * @return A new {@link Document} with the {@link Layers#POS_TAGS} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * sentence layer or the token layer is absent, or a token lies outside every - * sentence. + * sentence layer or the token layer is absent, a token lies outside every + * sentence, or the tagger does not return one tag per token of a sentence. */ @Override public Document annotate(Document document) { @@ -107,6 +107,10 @@ public Document annotate(Document document) { words[i] = tokens.get(first + i).value(); } final String[] tags = tagger.tag(words); + if (tags.length != count) { + throw new IllegalArgumentException( + "tagger returned " + tags.length + " tags for " + count + " tokens"); + } for (int i = 0; i < count; i++) { tagAnnotations.add(new Annotation<>(tokens.get(first + i).span(), tags[i])); } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java index 971215d898..e6d6b6a290 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test; +import opennlp.tools.namefind.NameSample; import opennlp.tools.namefind.TokenNameFinder; import opennlp.tools.util.Span; @@ -73,6 +74,11 @@ public void clearAdaptiveData() { assertEquals(1, cleared.get()); } + /** + * Verifies that a document carrying sentences but no token layer is rejected with a + * message naming the token layer, so the token check is exercised on its own rather + * than being shadowed by the sentence check. + */ @Test void testMissingTokenLayerThrows() { final TokenNameFinder finder = new TokenNameFinder() { @@ -86,8 +92,115 @@ public Span[] find(String[] tokens) { public void clearAdaptiveData() { } }; - assertThrows(IllegalArgumentException.class, - () -> new NameFinderAnnotator(finder).annotate(Document.of("no tokens"))); + final Document document = Document.of("no tokens") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "no tokens"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("document lacks the required layer tokens", e.getMessage()); + } + + /** + * Verifies that a mention the finder returns without a type is recorded with the + * {@link NameFinderAnnotator#UNTYPED} label, and that the label is the name-sample + * default type, so downstream consumers can rely on the two being interchangeable. + */ + @Test + void testUntypedMentionRecordedAsUntyped() { + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[] {new Span(0, 1)}; + } + + @Override + public void clearAdaptiveData() { + } + }; + final Document document = Document.of("Ana runs.") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."))); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + final List> entities = annotated.get(Layers.ENTITIES); + assertEquals(1, entities.size()); + assertEquals(NameSample.DEFAULT_TYPE, NameFinderAnnotator.UNTYPED); + assertEquals(NameFinderAnnotator.UNTYPED, entities.get(0).value()); + assertEquals(new Span(0, 3, NameFinderAnnotator.UNTYPED), entities.get(0).span()); + assertEquals(NameFinderAnnotator.UNTYPED, entities.get(0).span().getType()); + } + + /** + * Verifies that a mention whose token indices reach beyond its sentence's tokens is + * rejected loudly instead of silently taking its character span from the following + * sentence's tokens, and that the adaptive data is still cleared on that failure. + */ + @Test + void testMentionOutsideSentenceTokensFailsLoud() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + // two tokens in the sentence, but the mention claims three + return new Span[] {new Span(0, 3, "person")}; + } + + @Override + public void clearAdaptiveData() { + cleared.incrementAndGet(); + } + }; + final Document document = Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("finder returned mention [0..3) person outside the sentence's 2 tokens", + e.getMessage()); + assertEquals(1, cleared.get()); + } + + /** + * Verifies that a token lying outside every sentence is rejected loudly and that the + * adaptive data is still cleared on that failure, so a rejected document cannot leak + * finder state into the next one. + */ + @Test + void testTokenOutsideEverySentenceThrowsAndStillClears() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + cleared.incrementAndGet(); + } + }; + final Document document = Document.of("Ana runs. Bob") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + assertEquals(1, cleared.get()); } /** diff --git a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java index 95779fdd36..a7d44f1280 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java @@ -201,4 +201,42 @@ void testSentenceWithoutTokensContributesNothing() { assertEquals(List.of(List.of("The")), tagger.calls); assertEquals(1, annotated.get(Layers.POS_TAGS).size()); } + + /** + * Verifies that a tagger returning a wrong number of tags for a sentence is rejected + * loudly instead of silently misaligning the tag layer with the token layer. + */ + @Test + void testWrongTagCountFailsLoud() { + final POSTagger shortTagger = new POSTagger() { + + @Override + public String[] tag(String[] sentence) { + return new String[] {"X"}; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + }; + final Document document = Document.of("The dog") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 7), "The dog"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "The"), + new Annotation<>(new Span(4, 7), "dog"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new POSTaggerAnnotator(shortTagger).annotate(document)); + assertEquals("tagger returned 1 tags for 2 tokens", e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index 0561d667e4..9c8d5f5ff8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -28,8 +28,15 @@ import opennlp.tools.document.Layers; /** - * Adapts a {@link Lemmatizer} to the document pipeline: lemmatizes the token layer with - * its tags and provides {@link #LEMMAS}, one annotation per token on the token's span. + * Adapts a {@link Lemmatizer} to the document pipeline: reads {@link Layers#SENTENCES}, + * {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and provides {@link #LEMMAS}, one + * annotation per token on the token's span. + * + *

Each sentence is lemmatized separately, the way the lemmatizer contract expects its + * input, so lemmatization decisions never cross a sentence boundary. Token spans already + * refer to the original document text, so only the token and tag sequences handed to the + * lemmatizer are sliced per sentence; the produced lemma layer stays aligned with + * {@link Layers#TOKENS} by position.

* * @since 3.0.0 */ @@ -57,25 +64,35 @@ public LemmatizerAnnotator(Lemmatizer lemmatizer) { } /** - * Lemmatizes the token layer with its tags and adds the {@link #LEMMAS} layer. + * Lemmatizes the document sentence by sentence and adds the {@link #LEMMAS} layer. * - *

The required layers must be present, but they may be empty: a document without - * tokens yields a present-but-empty lemma layer.

+ *

For every sentence, the tokens whose spans lie inside the sentence span are + * lemmatized as one sequence together with their tags, and each lemma is emitted on + * its token's span. The required layers must be present, but they may be empty: a + * document without sentences or tokens yields a present-but-empty lemma layer, and a + * sentence containing no tokens contributes nothing.

* * @param document The document to annotate. Must not be {@code null} and must carry - * the {@link Layers#TOKENS} layer and a {@link Layers#POS_TAGS} layer - * of equal size. + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers and a + * {@link Layers#POS_TAGS} layer with exactly one tag per token, with + * every token lying inside a sentence. * @return A new {@link Document} with the {@link #LEMMAS} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * token layer or the tag layer is absent, the tag layer does not have exactly - * one tag per token, or the lemmatizer does not return one lemma per token. + * sentence layer, the token layer, or the tag layer is absent, the tag layer + * does not have exactly one tag per token, a token lies outside every + * sentence, or the lemmatizer does not return one lemma per token of a + * sentence. */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } + if (!document.layers().contains(Layers.SENTENCES)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.SENTENCES); + } if (!document.layers().contains(Layers.TOKENS)) { throw new IllegalArgumentException("document lacks the required layer " + Layers.TOKENS); @@ -84,33 +101,53 @@ public Document annotate(Document document) { throw new IllegalArgumentException("document lacks the required layer " + Layers.POS_TAGS); } + final List> sentences = document.get(Layers.SENTENCES); final List> tokens = document.get(Layers.TOKENS); final List> tags = document.get(Layers.POS_TAGS); if (tags.size() != tokens.size()) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } - final String[] words = new String[tokens.size()]; - final String[] posTags = new String[tokens.size()]; - for (int i = 0; i < words.length; i++) { - words[i] = tokens.get(i).value(); - posTags[i] = tags.get(i).value(); - } - final String[] lemmas = lemmatizer.lemmatize(words, posTags); - if (lemmas.length != words.length) { - throw new IllegalArgumentException( - "lemmatizer returned " + lemmas.length + " lemmas for " + words.length + " tokens"); + final List> layer = new ArrayList<>(tokens.size()); + // Walk the token layer once: both layers are in text order, so each sentence + // consumes the contiguous run of tokens whose spans it encloses. + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + final String[] posTags = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + posTags[i] = tags.get(first + i).value(); + } + final String[] lemmas = lemmatizer.lemmatize(words, posTags); + if (lemmas.length != count) { + throw new IllegalArgumentException( + "lemmatizer returned " + lemmas.length + " lemmas for " + count + " tokens"); + } + for (int i = 0; i < count; i++) { + layer.add(new Annotation<>(tokens.get(first + i).span(), lemmas[i])); + } } - final List> layer = new ArrayList<>(lemmas.length); - for (int i = 0; i < lemmas.length; i++) { - layer.add(new Annotation<>(tokens.get(i).span(), lemmas[i])); + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); } return document.with(LEMMAS, layer); } @Override public Set> requires() { - return Set.of(Layers.TOKENS, Layers.POS_TAGS); + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); } @Override diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java index 6a5596dfe8..0b8fe30def 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -17,7 +17,9 @@ package opennlp.tools.lemmatizer; +import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -49,6 +51,8 @@ public List> lemmatize(List toks, List tags) { @Test void testLemmasAlignWithTokens() { final Document document = Document.of("She ran home") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 12), "She ran home"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 3), "She"), new Annotation<>(new Span(4, 7), "ran"), @@ -67,6 +71,51 @@ void testLemmasAlignWithTokens() { Assertions.assertEquals("home", lemmas.get(2).value()); } + /** + * Verifies that the lemmatizer is invoked once per sentence with exactly that + * sentence's tokens and tags, so lemmatization decisions never see material from a + * neighboring sentence, and that the lemma layer still aligns with the token layer. + */ + @Test + void testLemmatizesPerSentence() { + final List> calls = new ArrayList<>(); + final Lemmatizer recording = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + calls.add(List.of(toks)); + return toks.clone(); + } + + @Override + public List> lemmatize(List toks, List tags) { + throw new UnsupportedOperationException(); + } + }; + final Document document = Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PROPN"), + new Annotation<>(new Span(4, 9), "VERB"), + new Annotation<>(new Span(10, 13), "PROPN"), + new Annotation<>(new Span(14, 19), "VERB"))); + + final Document lemmatized = new LemmatizerAnnotator(recording).annotate(document); + + Assertions.assertEquals(List.of( + List.of("Ana", "runs."), + List.of("Bob", "sits.")), calls); + Assertions.assertEquals(4, lemmatized.get(LemmatizerAnnotator.LEMMAS).size()); + Assertions.assertEquals(new Span(10, 13), + lemmatized.get(LemmatizerAnnotator.LEMMAS).get(2).span()); + } + @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, @@ -74,6 +123,7 @@ void testInvalidArguments() { final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); final Document misaligned = Document.of("a b") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "a b"))) .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))) .with(Layers.POS_TAGS, List.of()); Assertions.assertThrows(IllegalArgumentException.class, @@ -82,34 +132,104 @@ void testInvalidArguments() { /** * Verifies that a document lacking a required layer is rejected with a message naming - * the missing layer, instead of silently producing an empty lemma layer. + * the missing layer, for each of the three required layers in declaration order. */ @Test void testAbsentRequiredLayerThrowsWithExactMessage() { final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no layers"))); - Assertions.assertEquals("document lacks the required layer tokens", e.getMessage()); + Assertions.assertEquals("document lacks the required layer sentences", + e.getMessage()); + + final Document sentencesOnly = Document.of("a") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 1), "a"))); + final IllegalArgumentException tokenless = Assertions.assertThrows( + IllegalArgumentException.class, () -> annotator.annotate(sentencesOnly)); + Assertions.assertEquals("document lacks the required layer tokens", + tokenless.getMessage()); - final Document tokensOnly = Document.of("a") + final Document untagged = sentencesOnly .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))); final IllegalArgumentException tagless = Assertions.assertThrows( - IllegalArgumentException.class, () -> annotator.annotate(tokensOnly)); + IllegalArgumentException.class, () -> annotator.annotate(untagged)); Assertions.assertEquals("document lacks the required layer pos", tagless.getMessage()); } /** - * Verifies that present-but-empty token and tag layers yield a present-but-empty lemma - * layer rather than an exception. + * Verifies that present-but-empty layers yield a present-but-empty lemma layer rather + * than an exception. */ @Test void testEmptyPresentLayersYieldEmptyLemmaLayer() { final Document document = Document.of("") + .with(Layers.SENTENCES, List.of()) .with(Layers.TOKENS, List.of()) .with(Layers.POS_TAGS, List.of()); final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); Assertions.assertTrue(lemmatized.layers().contains(LemmatizerAnnotator.LEMMAS)); Assertions.assertTrue(lemmatized.get(LemmatizerAnnotator.LEMMAS).isEmpty()); } + + /** + * Verifies that a token lying outside every sentence is rejected loudly, matching the + * walk contract of the other per-sentence adapters. + */ + @Test + void testTokenOutsideEverySentenceThrowsWithExactMessage() { + final Document document = Document.of("Ana runs. Bob") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PROPN"), + new Annotation<>(new Span(4, 9), "VERB"), + new Annotation<>(new Span(10, 13), "PROPN"))); + final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(FIXTURE).annotate(document)); + Assertions.assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies that a lemmatizer returning a wrong number of lemmas for a sentence is + * rejected loudly instead of silently misaligning the lemma layer. + */ + @Test + void testWrongLemmaCountFailsLoud() { + final Lemmatizer shortLemmatizer = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + return new String[] {"a"}; + } + + @Override + public List> lemmatize(List toks, List tags) { + throw new UnsupportedOperationException(); + } + }; + final Document document = Document.of("a b") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "a b"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 1), "a"), + new Annotation<>(new Span(2, 3), "b"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 1), "X"), + new Annotation<>(new Span(2, 3), "X"))); + final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(shortLemmatizer).annotate(document)); + Assertions.assertEquals("lemmatizer returned 1 lemmas for 2 tokens", e.getMessage()); + } + + /** + * Verifies that the adapter declares all three consumed layers as required, so a + * pipeline without a sentence step fails at build time. + */ + @Test + void testRequiresSentencesTokensAndTags() { + Assertions.assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + new LemmatizerAnnotator(FIXTURE).requires()); + } } From 99e0b83fe695da6dc88884c06a446f06b96829d1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:14:51 -0400 Subject: [PATCH 06/92] OPENNLP-1888: Add StringUtil.isBlank following the toolkit whitespace definition A blank check under the toolkit's whitespace definition, which unlike String.isBlank covers the no-break spaces, so annotators validating labels and identifiers share one predicate instead of each carrying a private copy. Reads whole code points; tests pin the no-break and figure spaces, the empty string, and a supplementary-plane letter. --- .../java/opennlp/tools/util/StringUtil.java | 21 +++++++++++++++++++ .../opennlp/tools/util/StringUtilTest.java | 19 +++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 98cca59891..1842556720 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -268,6 +268,27 @@ public static boolean isEmpty(CharSequence theString) { return theString.length() == 0; } + /** + * Determines whether a {@link CharSequence} is blank: empty, or made up entirely of + * code points that {@link #isWhitespace(int)} accepts. Unlike + * {@link String#isBlank()}, this follows the toolkit's whitespace definition, which + * includes the no-break spaces the JDK predicate leaves out, so a value spelled + * entirely from them cannot pass a blank check as content. + * + * @param theString The {@link CharSequence} to examine. Must not be {@code null}. + * @return {@code true} if {@code theString} is empty or all whitespace. + */ + public static boolean isBlank(CharSequence theString) { + for (int i = 0; i < theString.length(); ) { + final int codePoint = Character.codePointAt(theString, i); + if (!isWhitespace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + /** * Get the minimum of three values. * diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java index a47306d3d4..73f81c209e 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java @@ -679,4 +679,23 @@ void testLowercaseBeyondBMP() { String lc = StringUtil.toLowerCase(input); Assertions.assertArrayEquals(expectedCodePoints, lc.codePoints().toArray()); } + + /** + * Verifies the blank check against the toolkit's whitespace definition: the + * no-break space is blank here although the JDK's own check does not cover it, + * whitespace-only and empty values are blank, and any non-whitespace code point, + * supplementary ones included, makes a value non-blank. + */ + @Test + void testIsBlankFollowsTheToolkitWhitespaceDefinition() { + Assertions.assertTrue(StringUtil.isBlank("")); + Assertions.assertTrue(StringUtil.isBlank(" \t\n")); + // U+00A0 no-break space and U+2007 figure space: JDK String.isBlank says false + Assertions.assertTrue(StringUtil.isBlank("\u00A0")); + Assertions.assertTrue(StringUtil.isBlank(" \u00A0\u2007 ")); + Assertions.assertFalse(StringUtil.isBlank("a")); + Assertions.assertFalse(StringUtil.isBlank(" a ")); + // U+10428, a supplementary-plane letter read as one code point, not two chars + Assertions.assertFalse(StringUtil.isBlank("\uD801\uDC28")); + } } From eb277fbb623bacece5d397fb9b9e10655d086a1d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:50:20 -0400 Subject: [PATCH 07/92] OPENNLP-1888: Manual chapter, review-convention pass, and key placement rule Adds the Document Annotation Container chapter to the manual, with every code example and every stated span and value mirroring the passing pipeline example test. The review pass aligns the branch with the project's conventions: layer key ids validate through StringUtil.isBlank, the annotator interface leaves thread safety implementation specific, the sentence and tokenizer adapters document annotate like their siblings, repeated rejection-message literals become per-class constants, and the name finder test's nine anonymous fixtures fold into one helper. Layers now states the key placement rule: core layer keys live there, capability layer keys on their providing annotator. --- .../tools/document/DocumentAnnotator.java | 5 +- .../java/opennlp/tools/document/LayerKey.java | 4 +- .../java/opennlp/tools/document/Layers.java | 5 + .../tools/document/NameFinderAnnotator.java | 7 +- .../tools/document/POSTaggerAnnotator.java | 7 +- .../document/SentenceDetectorAnnotator.java | 10 + .../tools/document/TokenizerAnnotator.java | 18 ++ .../document/NameFinderAnnotatorTest.java | 145 +++++--------- .../document/POSTaggerAnnotatorTest.java | 23 +-- .../tools/lemmatizer/LemmatizerAnnotator.java | 9 +- opennlp-docs/src/docbkx/document.xml | 179 ++++++++++++++++++ opennlp-docs/src/docbkx/opennlp.xml | 1 + 12 files changed, 282 insertions(+), 131 deletions(-) create mode 100644 opennlp-docs/src/docbkx/document.xml diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java index 3f44a6d856..6bc8417e6c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java @@ -25,9 +25,8 @@ * *

An annotator declares the layers it {@link #requires()} and {@link #provides()}, so * a {@link DocumentAnalyzer} can validate a pipeline before running it. Annotators are - * usually thin adapters over an existing analysis component and should hold no per-call - * state, so one instance can serve concurrent pipelines when the wrapped component - * allows it.

+ * usually thin adapters over an existing analysis component. Thread safety is + * implementation specific.

* * @since 3.0.0 */ diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java index 458a61b4c4..1f8ac378d4 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -19,6 +19,8 @@ import java.util.Objects; +import opennlp.tools.util.StringUtil; + /** * Identifies one annotation layer of a {@link Document} and carries the type of that * layer's annotation values, so reading a layer back is statically typed. @@ -56,7 +58,7 @@ private LayerKey(String id, Class type) { * {@code type} is {@code null}. */ public static LayerKey of(String id, Class type) { - if (id == null || id.isBlank()) { + if (id == null || StringUtil.isBlank(id)) { throw new IllegalArgumentException("id must not be null or blank"); } if (type == null) { diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index 99f0730550..f5808aed1c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -24,6 +24,11 @@ * producer may define further keys in its own package. New capabilities must never * require an addition here to function.

* + *

Placement rule: this class holds only the keys of the core linguistic layers + * every pipeline shares (sentences, tokens, tags, entities). A capability-specific + * layer's key lives on the annotator that provides it, for example the lemma layer's + * key on its adapter, so adding a capability never touches this class.

+ * * @since 3.0.0 */ public final class Layers { diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java index b129b77b29..75c33496c8 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -52,6 +52,9 @@ public class NameFinderAnnotator implements DocumentAnnotator { */ public static final String UNTYPED = NameSample.DEFAULT_TYPE; + /** The message prefix of every absent-required-layer rejection in this adapter. */ + private static final String MISSING_LAYER = "document lacks the required layer "; + private final TokenNameFinder finder; /** @@ -95,11 +98,11 @@ public Document annotate(Document document) { } final Set> present = document.layers(); if (!present.contains(Layers.SENTENCES)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.SENTENCES); } if (!present.contains(Layers.TOKENS)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.TOKENS); } final List> sentences = document.get(Layers.SENTENCES); diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java index f7564cd648..08e1b81f8c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -38,6 +38,9 @@ */ public class POSTaggerAnnotator implements DocumentAnnotator { + /** The message prefix of every absent-required-layer rejection in this adapter. */ + private static final String MISSING_LAYER = "document lacks the required layer "; + private final POSTagger tagger; /** @@ -78,11 +81,11 @@ public Document annotate(Document document) { } final Set> present = document.layers(); if (!present.contains(Layers.SENTENCES)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.SENTENCES); } if (!present.contains(Layers.TOKENS)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.TOKENS); } final List> sentences = document.get(Layers.SENTENCES); diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java index 47e7c58c34..282edf31dd 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java @@ -50,6 +50,16 @@ public SentenceDetectorAnnotator(SentenceDetector detector) { this.detector = detector; } + /** + * Detects sentences over the document text and adds the {@link Layers#SENTENCES} + * layer, each sentence annotated with its covered text on its span. + * + * @param document The document to annotate. Must not be {@code null}. + * @return A new {@link Document} with the {@link Layers#SENTENCES} layer added. + * Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or + * already carries the {@link Layers#SENTENCES} layer. + */ @Override public Document annotate(Document document) { if (document == null) { diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java index 5c21111a65..e9384bd7b9 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -52,6 +52,16 @@ public TokenizerAnnotator(Tokenizer tokenizer) { this.tokenizer = tokenizer; } + /** + * Tokenizes the document and adds the {@link Layers#TOKENS} layer, sentence by + * sentence when a sentence layer is present and over the whole text otherwise. + * + * @param document The document to annotate. Must not be {@code null}. + * @return A new {@link Document} with the {@link Layers#TOKENS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or + * already carries the {@link Layers#TOKENS} layer. + */ @Override public Document annotate(Document document) { if (document == null) { @@ -70,6 +80,14 @@ public Document annotate(Document document) { return document.with(Layers.TOKENS, tokens); } + /** + * Tokenizes one stretch of text and appends its tokens, shifted back into document + * coordinates. + * + * @param tokens The layer under construction. + * @param text The stretch to tokenize. + * @param offset The stretch's start offset in the document text. + */ private void addTokens(List> tokens, String text, int offset) { for (final Span span : tokenizer.tokenizePos(text)) { final Span shifted = new Span(span.getStart() + offset, span.getEnd() + offset); diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java index e6d6b6a290..09efc6ab54 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import org.junit.jupiter.api.Test; @@ -38,22 +39,39 @@ */ public class NameFinderAnnotatorTest { - @Test - void testTokenIndexSpansBecomeCharacterSpans() { - final AtomicInteger cleared = new AtomicInteger(); - final TokenNameFinder finder = new TokenNameFinder() { + /** + * Builds a finder over a fixed find function, with adaptive-data clearing counted + * in the given counter when one is supplied. + * + * @param find Maps a sentence's tokens to its mentions. + * @param cleared Counts {@code clearAdaptiveData} calls, or {@code null} to ignore. + * @return The finder fixture. Never {@code null}. + */ + private static TokenNameFinder finder(Function find, + AtomicInteger cleared) { + return new TokenNameFinder() { @Override public Span[] find(String[] tokens) { - // "New York" as a two-token person-free location mention - return new Span[] {new Span(1, 3, "location")}; + return find.apply(tokens); } @Override public void clearAdaptiveData() { - cleared.incrementAndGet(); + if (cleared != null) { + cleared.incrementAndGet(); + } } }; + } + + @Test + void testTokenIndexSpansBecomeCharacterSpans() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> { + // "New York" as a two-token person-free location mention + return new Span[] {new Span(1, 3, "location")}; + }, cleared); final Document document = Document.of("in New York today") .with(Layers.SENTENCES, List.of( @@ -81,17 +99,7 @@ public void clearAdaptiveData() { */ @Test void testMissingTokenLayerThrows() { - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - return new Span[0]; - } - - @Override - public void clearAdaptiveData() { - } - }; + final TokenNameFinder finder = finder(tokens -> new Span[0], null); final Document document = Document.of("no tokens") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "no tokens"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, @@ -106,17 +114,7 @@ public void clearAdaptiveData() { */ @Test void testUntypedMentionRecordedAsUntyped() { - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - return new Span[] {new Span(0, 1)}; - } - - @Override - public void clearAdaptiveData() { - } - }; + final TokenNameFinder finder = finder(tokens -> new Span[] {new Span(0, 1)}, null); final Document document = Document.of("Ana runs.") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) .with(Layers.TOKENS, List.of( @@ -140,19 +138,10 @@ public void clearAdaptiveData() { @Test void testMentionOutsideSentenceTokensFailsLoud() { final AtomicInteger cleared = new AtomicInteger(); - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - // two tokens in the sentence, but the mention claims three - return new Span[] {new Span(0, 3, "person")}; - } - - @Override - public void clearAdaptiveData() { - cleared.incrementAndGet(); - } - }; + final TokenNameFinder finder = finder(tokens -> { + // two tokens in the sentence, but the mention claims three + return new Span[] {new Span(0, 3, "person")}; + }, cleared); final Document document = Document.of("Ana runs. Bob sits.") .with(Layers.SENTENCES, List.of( new Annotation<>(new Span(0, 9), "Ana runs."), @@ -178,18 +167,7 @@ public void clearAdaptiveData() { @Test void testTokenOutsideEverySentenceThrowsAndStillClears() { final AtomicInteger cleared = new AtomicInteger(); - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - return new Span[0]; - } - - @Override - public void clearAdaptiveData() { - cleared.incrementAndGet(); - } - }; + final TokenNameFinder finder = finder(tokens -> new Span[0], cleared); final Document document = Document.of("Ana runs. Bob") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "Ana runs."))) .with(Layers.TOKENS, List.of( @@ -213,20 +191,11 @@ public void clearAdaptiveData() { void testFindsPerSentenceAndMapsSentenceLocalIndices() { final List> calls = new ArrayList<>(); final AtomicInteger cleared = new AtomicInteger(); - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - calls.add(List.of(tokens)); - // the first token of every sentence is a person mention, in sentence-local indices - return new Span[] {new Span(0, 1, "person")}; - } - - @Override - public void clearAdaptiveData() { - cleared.incrementAndGet(); - } - }; + final TokenNameFinder finder = finder(tokens -> { + calls.add(List.of(tokens)); + // the first token of every sentence is a person mention, in sentence-local indices + return new Span[] {new Span(0, 1, "person")}; + }, cleared); final Document document = Document.of("Ana runs. Bob sits.") .with(Layers.SENTENCES, List.of( @@ -259,17 +228,7 @@ public void clearAdaptiveData() { */ @Test void testRequiresSentencesAndTokens() { - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - return new Span[0]; - } - - @Override - public void clearAdaptiveData() { - } - }; + final TokenNameFinder finder = finder(tokens -> new Span[0], null); assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), new NameFinderAnnotator(finder).requires()); } @@ -281,18 +240,10 @@ public void clearAdaptiveData() { @Test void testEmptyPresentLayersYieldEmptyEntityLayer() { final AtomicInteger found = new AtomicInteger(); - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - found.incrementAndGet(); - return new Span[0]; - } - - @Override - public void clearAdaptiveData() { - } - }; + final TokenNameFinder finder = finder(tokens -> { + found.incrementAndGet(); + return new Span[0]; + }, null); final Document document = Document.of("") .with(Layers.SENTENCES, List.of()) .with(Layers.TOKENS, List.of()); @@ -310,17 +261,7 @@ public void clearAdaptiveData() { */ @Test void testAbsentSentenceLayerThrowsWithExactMessage() { - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - return new Span[0]; - } - - @Override - public void clearAdaptiveData() { - } - }; + final TokenNameFinder finder = finder(tokens -> new Span[0], null); final Document document = Document.of("Ana") .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "Ana"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, diff --git a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java index a7d44f1280..719bb7b6da 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java @@ -41,9 +41,10 @@ public class POSTaggerAnnotatorTest { /** * A tagger that records the exact token sequence of every call and answers with one - * {@code "X"} tag per token, so the per-call slicing is observable. + * {@code "X"} tag per token, so the per-call slicing is observable. Tests override + * {@link #tag(String[])} where a deviant answer is the fixture. */ - private static final class RecordingTagger implements POSTagger { + private static class RecordingTagger implements POSTagger { private final List> calls = new ArrayList<>(); @@ -208,27 +209,13 @@ void testSentenceWithoutTokensContributesNothing() { */ @Test void testWrongTagCountFailsLoud() { - final POSTagger shortTagger = new POSTagger() { + // one tag regardless of sentence length, so a two-token sentence trips the check + final POSTagger shortTagger = new RecordingTagger() { @Override public String[] tag(String[] sentence) { return new String[] {"X"}; } - - @Override - public String[] tag(String[] sentence, Object[] additionalContext) { - return tag(sentence); - } - - @Override - public Sequence[] topKSequences(String[] sentence) { - throw new UnsupportedOperationException("the adapter only calls tag"); - } - - @Override - public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { - throw new UnsupportedOperationException("the adapter only calls tag"); - } }; final Document document = Document.of("The dog") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 7), "The dog"))) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index 9c8d5f5ff8..d4240ec2cf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -48,6 +48,9 @@ public class LemmatizerAnnotator implements DocumentAnnotator { */ public static final LayerKey LEMMAS = LayerKey.of("lemmas", String.class); + /** The message prefix of every absent-required-layer rejection in this adapter. */ + private static final String MISSING_LAYER = "document lacks the required layer "; + private final Lemmatizer lemmatizer; /** @@ -90,15 +93,15 @@ public Document annotate(Document document) { throw new IllegalArgumentException("document must not be null"); } if (!document.layers().contains(Layers.SENTENCES)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.SENTENCES); } if (!document.layers().contains(Layers.TOKENS)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.TOKENS); } if (!document.layers().contains(Layers.POS_TAGS)) { - throw new IllegalArgumentException("document lacks the required layer " + throw new IllegalArgumentException(MISSING_LAYER + Layers.POS_TAGS); } final List> sentences = document.get(Layers.SENTENCES); diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml new file mode 100644 index 0000000000..99b1209539 --- /dev/null +++ b/opennlp-docs/src/docbkx/document.xml @@ -0,0 +1,179 @@ + + + + + + + Document Annotation Container + +
+ Introduction + + The package opennlp.tools.document provides an immutable container + that carries the original text of one document together with any number of typed + annotation layers over it. Every annotation is anchored to a + Span of the text exactly as the caller supplied it, never to a + normalized or otherwise derived form, so any result of any pipeline step can be + highlighted in the source text. + + + The design follows three rules: + + + + + Offset-anchored. A layer is a list of + Annotation values, each pairing a Span in original + text coordinates with a typed value. Annotations reference other annotations + by index within their layer, never by object identity. + + + + + Open key space. A layer is identified by a + LayerKey that also carries the type of its values, so reading a + layer back is statically typed. Two keys are equal when both their id and + their value type are equal, so independently created constants for the same + layer interoperate. The keys of the core linguistic layers live in + Layers; a capability-specific layer's key lives on the annotator + that provides it, and adding a capability never changes the container. + + + + + Immutable. Adding a layer returns a new + document that shares the unchanged layers with its ancestor. Documents are + safe to share between threads. + + + +
+ +
+ Building a pipeline + + A pipeline step implements DocumentAnnotator: it reads layers from a + document and returns a new document with its own layers added. An annotator + declares the layers it requires and provides, and a + DocumentAnalyzer validates those declarations when the pipeline is + assembled: every required layer must be provided by an earlier annotator, and no + two annotators may provide the same layer, so a misordered pipeline fails at + build time rather than midway through a document. + + + Adapters for the toolkit's own components are provided: + SentenceDetectorAnnotator, TokenizerAnnotator, + POSTaggerAnnotator, NameFinderAnnotator, + LemmatizerAnnotator, and StemmerAnnotator. Each wraps + the existing single-task API, which stays the primary API for single-task use. + The following pipeline combines three adapters with one custom annotator and + analyzes a two-sentence text; the components behind the adapters are any + SentenceDetector, Tokenizer, and + POSTagger, for example the ME implementations loaded from models: + + + + + + The resulting document carries exactly the four layers the pipeline provides. + The sentence layer holds two annotations, [0..14) covering + The dog barks. and [15..23) covering + It naps.. The token layer holds five tokens whose spans refer to + the document text even inside the second sentence, so + It is [15..17) and naps. is + [18..23). Layers produced per token stay aligned with the token + layer by position: + + + > tokens = document.get(Layers.TOKENS); +List> tags = document.get(Layers.POS_TAGS); +for (int i = 0; i < tags.size(); i++) { + // each tag sits on its token's span, e.g. "DT" on [0..3) for "The" + Span span = tags.get(i).span(); +} + +// every span refers to the original text, so covered text round-trips +for (Annotation token : tokens) { + CharSequence covered = token.span().getCoveredText(document.text()); +}]]> + +
+ +
+ Writing a custom annotator + + A new capability contributes its results as one more layer without any change to + the container. The annotator below defines its own key in its own code, reads + the token layer, and provides one integer annotation per token; the analyzer's + build-time validation guarantees a tokenizer ran earlier: + + + TOKEN_LENGTHS = + LayerKey.of("token-lengths", Integer.class); + +class TokenLengthAnnotator implements DocumentAnnotator { + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + List> tokens = document.get(Layers.TOKENS); + List> lengths = new ArrayList<>(tokens.size()); + for (Annotation token : tokens) { + lengths.add(new Annotation<>(token.span(), token.value().length())); + } + return document.with(TOKEN_LENGTHS, lengths); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set> provides() { + return Set.of(TOKEN_LENGTHS); + } +}]]> + + + Reading the layer back is statically typed by the key, so the values are used as + numbers without a cast; for the text above the five values are + 3, 3, 6, 2, 5, each on its token's span: + + + > lengths = document.get(TOKEN_LENGTHS); +int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> + + + Required layers must be present, but they may be empty: an empty token layer + yields the annotator's provided layers present but empty, so a pipeline degrades + gracefully on documents without content. An absent required layer is rejected + with an IllegalArgumentException naming the layer, because a + missing pipeline stage is an assembly error, not an empty document. + +
+ diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 36641c2c89..0761fc95ff 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -109,6 +109,7 @@ under the License. + From 1ba2fcdae336fbb18da55ff1148e42193377ab5b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 09:28:31 -0400 Subject: [PATCH 08/92] OPENNLP-1888: Namespace the standard layer key ids Every key the toolkit defines now carries the opennlp: id prefix (opennlp:sentences, opennlp:tokens, opennlp:pos, opennlp:entities, opennlp:lemmas, opennlp:stems). An extension defines its keys under its own prefix, and a bare id stays legal for an application-local layer, so ids from independent producers cannot collide. The rule is stated on Layers, LayerKey, and in the manual chapter. --- .../main/java/opennlp/tools/document/LayerKey.java | 6 ++++-- .../main/java/opennlp/tools/document/Layers.java | 13 +++++++++---- .../tools/document/DocumentContractTest.java | 2 +- .../tools/document/NameFinderAnnotatorTest.java | 4 ++-- .../tools/document/POSTaggerAnnotatorTest.java | 4 ++-- .../tools/lemmatizer/LemmatizerAnnotator.java | 2 +- .../opennlp/tools/stemmer/StemmerAnnotator.java | 2 +- .../tools/lemmatizer/LemmatizerAnnotatorTest.java | 6 +++--- .../opennlp/tools/stemmer/StemmerAnnotatorTest.java | 2 +- opennlp-docs/src/docbkx/document.xml | 6 +++++- 10 files changed, 29 insertions(+), 18 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java index 1f8ac378d4..fdaaac84a1 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -48,8 +48,10 @@ private LayerKey(String id, Class type) { /** * Creates a {@link LayerKey}. * - * @param id The layer identifier, for example {@code tokens}. Must not be {@code null} - * or blank. + * @param id The layer identifier, for example {@code opennlp:tokens}. Keys defined by + * the toolkit carry the {@code opennlp:} prefix, an extension uses its own + * prefix, and a bare id is legal for an application-local layer. Must not + * be {@code null} or blank. * @param type The class of the annotation values stored under the key. Must not be * {@code null}. * @param The type of the annotation values. diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index f5808aed1c..75fd4cd8d5 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -24,6 +24,11 @@ * producer may define further keys in its own package. New capabilities must never * require an addition here to function.

* + *

Namespace rule: every key the toolkit itself defines carries the + * {@code opennlp:} id prefix. An extension defines its keys under its own prefix, and + * a bare id without a prefix is legal for an application-local layer, so ids from + * independent producers cannot collide.

+ * *

Placement rule: this class holds only the keys of the core linguistic layers * every pipeline shares (sentences, tokens, tags, entities). A capability-specific * layer's key lives on the annotator that provides it, for example the lemma layer's @@ -36,23 +41,23 @@ public final class Layers { /** * Sentence boundaries; each annotation covers one sentence and carries its text. */ - public static final LayerKey SENTENCES = LayerKey.of("sentences", String.class); + public static final LayerKey SENTENCES = LayerKey.of("opennlp:sentences", String.class); /** * Token boundaries; each annotation covers one token and carries its text. */ - public static final LayerKey TOKENS = LayerKey.of("tokens", String.class); + public static final LayerKey TOKENS = LayerKey.of("opennlp:tokens", String.class); /** * Part-of-speech tags; one annotation per token, aligned with {@link #TOKENS} by * position, carrying the tag. */ - public static final LayerKey POS_TAGS = LayerKey.of("pos", String.class); + public static final LayerKey POS_TAGS = LayerKey.of("opennlp:pos", String.class); /** * Named entities; each annotation covers one mention and carries the entity type. */ - public static final LayerKey ENTITIES = LayerKey.of("entities", String.class); + public static final LayerKey ENTITIES = LayerKey.of("opennlp:entities", String.class); private Layers() { // This class holds constants only and is never instantiated. diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index f46701ef12..f71b590c16 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -234,7 +234,7 @@ public String toString() { final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder().add(needsTags); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, builder::build); - assertEquals("annotator tag-consumer requires layer pos," + assertEquals("annotator tag-consumer requires layer opennlp:pos," + " which no earlier annotator provides", e.getMessage()); } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java index 09efc6ab54..99f80dae03 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -104,7 +104,7 @@ void testMissingTokenLayerThrows() { .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 9), "no tokens"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(finder).annotate(document)); - assertEquals("document lacks the required layer tokens", e.getMessage()); + assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); } /** @@ -266,6 +266,6 @@ void testAbsentSentenceLayerThrowsWithExactMessage() { .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "Ana"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(finder).annotate(document)); - assertEquals("document lacks the required layer sentences", e.getMessage()); + assertEquals("document lacks the required layer opennlp:sentences", e.getMessage()); } } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java index 719bb7b6da..ccd7b0cfe0 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java @@ -152,7 +152,7 @@ void testAbsentSentenceLayerThrowsWithExactMessage() { new Annotation<>(new Span(4, 7), "dog"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); - assertEquals("document lacks the required layer sentences", e.getMessage()); + assertEquals("document lacks the required layer opennlp:sentences", e.getMessage()); } /** @@ -165,7 +165,7 @@ void testAbsentTokenLayerThrowsWithExactMessage() { .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 7), "The dog"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new POSTaggerAnnotator(new RecordingTagger()).annotate(document)); - assertEquals("document lacks the required layer tokens", e.getMessage()); + assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index d4240ec2cf..c3f5897d6b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -46,7 +46,7 @@ public class LemmatizerAnnotator implements DocumentAnnotator { * The lemma layer. It is aligned with the token layer by position, and each annotation * carries the lemma of its token on that token's span. */ - public static final LayerKey LEMMAS = LayerKey.of("lemmas", String.class); + public static final LayerKey LEMMAS = LayerKey.of("opennlp:lemmas", String.class); /** The message prefix of every absent-required-layer rejection in this adapter. */ private static final String MISSING_LAYER = "document lacks the required layer "; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index 330087f5d8..015779ee07 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -42,7 +42,7 @@ public class StemmerAnnotator implements DocumentAnnotator { * The stem layer. It is aligned with the token layer by position, and each annotation * carries the stem of its token on that token's span. */ - public static final LayerKey STEMS = LayerKey.of("stems", String.class); + public static final LayerKey STEMS = LayerKey.of("opennlp:stems", String.class); private final Stemmer stemmer; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java index 0b8fe30def..d31f092104 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -139,21 +139,21 @@ void testAbsentRequiredLayerThrowsWithExactMessage() { final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no layers"))); - Assertions.assertEquals("document lacks the required layer sentences", + Assertions.assertEquals("document lacks the required layer opennlp:sentences", e.getMessage()); final Document sentencesOnly = Document.of("a") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 1), "a"))); final IllegalArgumentException tokenless = Assertions.assertThrows( IllegalArgumentException.class, () -> annotator.annotate(sentencesOnly)); - Assertions.assertEquals("document lacks the required layer tokens", + Assertions.assertEquals("document lacks the required layer opennlp:tokens", tokenless.getMessage()); final Document untagged = sentencesOnly .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))); final IllegalArgumentException tagless = Assertions.assertThrows( IllegalArgumentException.class, () -> annotator.annotate(untagged)); - Assertions.assertEquals("document lacks the required layer pos", + Assertions.assertEquals("document lacks the required layer opennlp:pos", tagless.getMessage()); } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java index 86b3a82ec7..3f84b5303e 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -63,7 +63,7 @@ void testAbsentTokenLayerThrowsWithExactMessage() { final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no tokens"))); - Assertions.assertEquals("document lacks the required layer tokens", e.getMessage()); + Assertions.assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); } /** diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 99b1209539..980bd44d09 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -47,7 +47,11 @@ their value type are equal, so independently created constants for the same layer interoperate. The keys of the core linguistic layers live in Layers; a capability-specific layer's key lives on the annotator - that provides it, and adding a capability never changes the container. + that provides it, and adding a capability never changes the container. Ids + are namespaced: keys the toolkit defines carry the opennlp: + prefix, an extension uses its own prefix, and a bare id, like the + token-lengths key below, is legal for an application-local + layer. From f8b2ee97ebd3e150f20a82e535b14dfc4bfb7ce9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 09:32:17 -0400 Subject: [PATCH 09/92] OPENNLP-1888: Declare per-key positional or document scope A layer key now declares whether its layer is positional or document-scoped. A positional key, the default, guarantees a span on every annotation, so consumers never null-check one. A document-scoped key, created through LayerKey.document, carries whole-document values without spans, the home for a language id, a category distribution, or provenance. The scope is declared per key, never per annotation: the container rejects a span-less annotation under a positional key and a spanned annotation under a document-scoped key, naming the layer either way. Scope participates in key equality. --- .../opennlp/tools/document/Annotation.java | 35 ++++++--- .../java/opennlp/tools/document/Document.java | 10 ++- .../tools/document/ImmutableDocument.java | 15 +++- .../java/opennlp/tools/document/LayerKey.java | 74 ++++++++++++++++--- .../tools/document/DocumentContractTest.java | 39 ++++++++++ .../document/DocumentPipelineExampleTest.java | 16 ++++ .../opennlp/tools/document/DocumentTest.java | 8 +- opennlp-docs/src/docbkx/document.xml | 19 ++++- 8 files changed, 186 insertions(+), 30 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java index eaed6bf7f4..3d3aaa1fd0 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java @@ -21,16 +21,20 @@ /** * One annotation of a {@link Document}: a typed value anchored to a {@link Span} of the - * document's original text. + * document's original text, or a span-less value under a + * {@link LayerKey.Scope#DOCUMENT document-scoped} key. * *

The span always refers to the text the document was created with, never to a * normalized or otherwise derived form, so any annotation can be highlighted in what the - * caller supplied. Annotations that need to reference other annotations, for example a - * dependency arc naming its head token, do so by the index of the target annotation - * within its layer, never by object identity.

+ * caller supplied. Whether a span is present is decided by the layer key's scope, not + * per annotation: the container rejects a span-less annotation under a positional key + * and a spanned annotation under a document-scoped key. Annotations that need to + * reference other annotations, for example a dependency arc naming its head token, do + * so by the index of the target annotation within its layer, never by object + * identity.

* - * @param span The location of the annotation in the original text. Must not be - * {@code null}. + * @param span The location of the annotation in the original text, or {@code null} for + * a value under a document-scoped key. * @param value The annotation value. Must not be {@code null}. * @param The type of the annotation value. * @@ -41,15 +45,24 @@ public record Annotation(Span span, T value) { /** * Validates the annotation. * - * @throws IllegalArgumentException Thrown if {@code span} or {@code value} is - * {@code null}. + * @throws IllegalArgumentException Thrown if {@code value} is {@code null}. */ public Annotation { - if (span == null) { - throw new IllegalArgumentException("span must not be null"); - } if (value == null) { throw new IllegalArgumentException("value must not be null"); } } + + /** + * Creates a span-less annotation for a {@link LayerKey.Scope#DOCUMENT + * document-scoped} layer. + * + * @param value The annotation value. Must not be {@code null}. + * @param The type of the annotation value. + * @return An {@link Annotation} without a span. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code value} is {@code null}. + */ + public static Annotation of(T value) { + return new Annotation<>(null, value); + } } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 53453ff856..2c0a2f2aec 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -28,7 +28,9 @@ * {@link LayerKey}. The container itself knows nothing about specific layers; every * analysis capability contributes its results as one more layer without any change to * this interface, which is what keeps new capabilities additive. All spans refer to - * {@link #text()} as supplied, never to a derived form.

+ * {@link #text()} as supplied, never to a derived form. A + * {@link LayerKey.Scope#DOCUMENT document-scoped} layer carries whole-document values + * without spans, for example a language id.

* *

Documents are immutable: {@link #with(LayerKey, List)} returns a new document that * shares the unchanged layers. Instances are safe to share between threads.

@@ -76,8 +78,10 @@ static Document of(CharSequence text) { * @param layer The key of the layer to add. Must not be {@code null} and must not * already be present. * @param annotations The annotations of the layer. Must not be {@code null}, must not - * contain {@code null}, every value must be assignable to the - * layer's type, and every span must lie within the text bounds. + * contain {@code null}, and every value must be assignable to the + * layer's type. Under a positional key every annotation must carry + * a span within the text bounds; under a document-scoped key no + * annotation may carry a span. * @param The type of the layer's annotation values. * @return A new {@link Document} sharing this document's text and existing layers. * Never {@code null}. diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index 43276b4cef..e2ca8cd5f3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -96,9 +96,18 @@ public Document with(LayerKey layer, List> annotations) { + annotation.value().getClass().getName() + " does not match layer " + layer); } final Span span = annotation.span(); - if (span.getEnd() > text.length()) { - throw new IllegalArgumentException("span " + span + " exceeds the text length " - + text.length() + " in layer " + layer); + if (layer.scope() == LayerKey.Scope.POSITIONAL) { + if (span == null) { + throw new IllegalArgumentException( + "positional layer " + layer + " requires a span on every annotation"); + } + if (span.getEnd() > text.length()) { + throw new IllegalArgumentException("span " + span + " exceeds the text length " + + text.length() + " in layer " + layer); + } + } else if (span != null) { + throw new IllegalArgumentException( + "document-scoped layer " + layer + " must not carry spans"); } } final Map, List>> grown = new LinkedHashMap<>(layers); diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java index fdaaac84a1..7d72411bd5 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -26,10 +26,16 @@ * layer's annotation values, so reading a layer back is statically typed. * *

The key space is deliberately open: any producer may define new keys in its own - * package, and the container never enumerates them. Two keys are equal when both their - * id and their value type are equal, so independently created constants for the same - * layer interoperate. Standard keys for the toolkit's own results live in - * {@link Layers}.

+ * package, and the container never enumerates them. Two keys are equal when their id, + * their value type, and their {@link Scope} are equal, so independently created + * constants for the same layer interoperate. Standard keys for the toolkit's own + * results live in {@link Layers}.

+ * + *

A key declares its {@link Scope}: a {@link Scope#POSITIONAL positional} key + * guarantees a span on every annotation, and a {@link Scope#DOCUMENT document-scoped} + * key carries whole-document values without spans, for example a language id or a + * category distribution. The scope is declared per key, never per annotation, so + * consumers of a positional layer never null-check a span.

* * @param The type of the annotation values stored under this key. * @@ -37,12 +43,22 @@ */ public final class LayerKey { + /** How the annotations of a layer relate to the document text. */ + public enum Scope { + /** Every annotation of the layer is anchored to a span of the text. */ + POSITIONAL, + /** The layer's values describe the document as a whole and carry no spans. */ + DOCUMENT + } + private final String id; private final Class type; + private final Scope scope; - private LayerKey(String id, Class type) { + private LayerKey(String id, Class type, Scope scope) { this.id = id; this.type = type; + this.scope = scope; } /** @@ -55,18 +71,51 @@ private LayerKey(String id, Class type) { * @param type The class of the annotation values stored under the key. Must not be * {@code null}. * @param The type of the annotation values. - * @return A {@link LayerKey}. Never {@code null}. + * @return A {@link Scope#POSITIONAL positional} {@link LayerKey}. Never {@code null}. * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or * {@code type} is {@code null}. */ public static LayerKey of(String id, Class type) { + return key(id, type, Scope.POSITIONAL); + } + + /** + * Creates a {@link Scope#DOCUMENT document-scoped} {@link LayerKey} for values that + * describe the document as a whole, for example a language id, a category + * distribution, or provenance. Annotations under such a key carry no span. + * + * @param id The layer identifier, following the same prefix rules as + * {@link #of(String, Class)}. Must not be {@code null} or blank. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A document-scoped {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + public static LayerKey document(String id, Class type) { + return key(id, type, Scope.DOCUMENT); + } + + /** + * Validates the components and creates the key. + * + * @param id The layer identifier. + * @param type The value class. + * @param scope The declared scope. + * @param The type of the annotation values. + * @return The {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + private static LayerKey key(String id, Class type, Scope scope) { if (id == null || StringUtil.isBlank(id)) { throw new IllegalArgumentException("id must not be null or blank"); } if (type == null) { throw new IllegalArgumentException("type must not be null"); } - return new LayerKey<>(id, type); + return new LayerKey<>(id, type, scope); } /** @@ -83,6 +132,13 @@ public Class type() { return type; } + /** + * @return The declared scope of the layer. Never {@code null}. + */ + public Scope scope() { + return scope; + } + @Override public boolean equals(Object obj) { if (this == obj) { @@ -91,12 +147,12 @@ public boolean equals(Object obj) { if (!(obj instanceof LayerKey other)) { return false; } - return id.equals(other.id) && type.equals(other.type); + return id.equals(other.id) && type.equals(other.type) && scope == other.scope; } @Override public int hashCode() { - return Objects.hash(id, type); + return Objects.hash(id, type, scope); } @Override diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index f71b590c16..3d0ab766d3 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -28,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -83,6 +84,44 @@ void testKeysWithSameIdButDifferentTypesAreDifferentLayers() { assertEquals(7, document.get(asInteger).get(0).value()); } + /** + * Verifies the per-key scope contract: a document-scoped layer carries span-less + * values and round-trips them, a spanned annotation under a document-scoped key is + * rejected with a message naming the layer, a span-less annotation under a + * positional key is rejected likewise, and two keys differing only in scope are + * unequal and never address the same layer. + */ + @Test + void testDocumentScopedLayersCarrySpanlessValues() { + final LayerKey language = LayerKey.document("language", String.class); + assertEquals(LayerKey.Scope.DOCUMENT, language.scope()); + assertEquals(LayerKey.Scope.POSITIONAL, WORDS.scope()); + + final Document document = Document.of("the dog") + .with(language, List.of(Annotation.of("eng"))); + assertEquals(1, document.get(language).size()); + assertEquals("eng", document.get(language).get(0).value()); + assertNull(document.get(language).get(0).span()); + + final IllegalArgumentException spanned = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(language, + List.of(new Annotation<>(new Span(0, 3), "eng")))); + assertEquals("document-scoped layer language must not carry spans", + spanned.getMessage()); + + final IllegalArgumentException spanless = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(WORDS, List.of(Annotation.of("the")))); + assertEquals("positional layer words requires a span on every annotation", + spanless.getMessage()); + + final LayerKey positionalTwin = LayerKey.of("language", String.class); + assertNotEquals(language, positionalTwin); + final Document both = Document.of("the") + .with(language, List.of(Annotation.of("eng"))) + .with(positionalTwin, List.of(new Annotation<>(new Span(0, 3), "the"))); + assertEquals(2, both.layers().size()); + } + /** * Verifies that a layer preserves the insertion order of its annotations: the * container does not sort by span, so a producer that wants span order must supply diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java index 31a2379380..14eeda39a4 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java @@ -31,6 +31,7 @@ import opennlp.tools.util.Span; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; /** * Walks through the document pipeline the way a first-time user would: wrap existing @@ -190,6 +191,21 @@ public Set> provides() { } } + /** + * Mirrors the manual's document-scoped layer example: a language id rides a + * document-scoped key as a span-less value with exactly the id, value, and null span + * the chapter shows. + */ + @Test + void testDocumentScopedLayerExample() { + final LayerKey language = LayerKey.document("app:language", String.class); + final Document document = Document.of("The dog barks. It naps."); + + final Document tagged = document.with(language, List.of(Annotation.of("eng"))); + assertEquals("eng", tagged.get(language).get(0).value()); + assertNull(tagged.get(language).get(0).span()); + } + /** * Runs the full pipeline story: sentences, tokens, part-of-speech tags, and one custom * layer over a two-sentence text, then verifies every annotation of every layer, span diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java index 5af4908bc8..846007652f 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java @@ -112,8 +112,11 @@ void testNullArgumentsThrow() { @Test void testAnnotationValidation() { - assertThrows(IllegalArgumentException.class, () -> new Annotation<>(null, "the")); + // a span-less annotation is legal to build; the container judges it per key scope assertThrows(IllegalArgumentException.class, () -> new Annotation<>(new Span(0, 3), null)); + assertThrows(IllegalArgumentException.class, () -> Annotation.of(null)); + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(WORDS, List.of(Annotation.of("the")))); } @Test @@ -121,5 +124,8 @@ void testLayerKeyValidation() { assertThrows(IllegalArgumentException.class, () -> LayerKey.of(" ", String.class)); assertThrows(IllegalArgumentException.class, () -> LayerKey.of(null, String.class)); assertThrows(IllegalArgumentException.class, () -> LayerKey.of("words", null)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.document(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.document(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.document("lang", null)); } } diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 980bd44d09..b8ef351851 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -43,9 +43,9 @@ Open key space. A layer is identified by a LayerKey that also carries the type of its values, so reading a - layer back is statically typed. Two keys are equal when both their id and - their value type are equal, so independently created constants for the same - layer interoperate. The keys of the core linguistic layers live in + layer back is statically typed. Two keys are equal when their id, their + value type, and their scope are equal, so independently created constants + for the same layer interoperate. The keys of the core linguistic layers live in Layers; a capability-specific layer's key lives on the annotator that provides it, and adding a capability never changes the container. Ids are namespaced: keys the toolkit defines carry the opennlp: @@ -62,6 +62,19 @@ + + A key declares its scope. Keys are positional by default: every annotation + carries a span, and consumers never null-check it. A document-scoped key + carries whole-document values without spans, which is where a language id, + a category distribution, or provenance belongs: + + + LANGUAGE = LayerKey.document("app:language", String.class); + +Document tagged = document.with(LANGUAGE, List.of(Annotation.of("eng"))); +String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]> +
From b39dd25d1dfb921391fd38185908ba1006b69ae3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 09:32:40 -0400 Subject: [PATCH 10/92] OPENNLP-1888: State the index-reference invariants in the specification text The three invariants the contract tests already enforce are now stated on the Document interface and in the manual chapter: layers preserve insertion order and are never reordered, layers are immutable once added and detached from the caller's input list, and adding a layer is once-only with the rejection naming the key. Together they keep index-based references between layers valid for the lifetime of the document. --- .../src/main/java/opennlp/tools/document/Document.java | 9 +++++++++ opennlp-docs/src/docbkx/document.xml | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 2c0a2f2aec..dc4be9616c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -35,6 +35,15 @@ *

Documents are immutable: {@link #with(LayerKey, List)} returns a new document that * shares the unchanged layers. Instances are safe to share between threads.

* + *

Three invariants make index-based references sound. A layer preserves its + * insertion order, and the container never sorts or reorders it. A layer is immutable + * once added: the returned lists reject modification and are detached from the + * caller's input list. Providing a layer that already exists is rejected loudly: the + * add is once-only, and the exception names the offending key. An annotation that + * references another annotation by its index within a layer, for example a dependency + * arc naming its head token, therefore stays valid for the lifetime of the + * document.

+ * * @since 3.0.0 */ public interface Document { diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index b8ef351851..bf9b4b10f9 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -58,7 +58,11 @@ Immutable. Adding a layer returns a new document that shares the unchanged layers with its ancestor. Documents are - safe to share between threads. + safe to share between threads. Three invariants make index references + sound: a layer preserves its insertion order and is never sorted or + reordered, a layer is immutable once added and detached from the caller's + input list, and providing a layer that already exists is rejected, so an + index reference stays valid for the lifetime of the document. From e4d6ed7189f167753c83e52adc90df991d1a7ff7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 09:33:29 -0400 Subject: [PATCH 11/92] OPENNLP-1888: Document the gold-layer convention A corpus may carry a hand-annotated version of a layer beside a produced one. The convention is a gold: id prefix on the same key scheme, for example gold:opennlp:tokens beside opennlp:tokens. Because adding a layer is once-only, competing versions of a layer always live under distinct keys and never replace each other. Stated on Layers and in the manual chapter, with a contract test pinning the coexistence. --- .../java/opennlp/tools/document/Layers.java | 6 ++++++ .../tools/document/DocumentContractTest.java | 20 +++++++++++++++++++ opennlp-docs/src/docbkx/document.xml | 7 +++++++ 3 files changed, 33 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index 75fd4cd8d5..cec6960eb2 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -29,6 +29,12 @@ * a bare id without a prefix is legal for an application-local layer, so ids from * independent producers cannot collide.

* + *

Gold versus predicted: a corpus may carry a hand-annotated version of a layer + * beside a produced one. The convention is a {@code gold:} id prefix on the same key + * scheme, for example {@code gold:opennlp:tokens} beside {@code opennlp:tokens}. + * Because adding a layer is once-only, competing versions of a layer always live under + * distinct keys and never replace each other.

+ * *

Placement rule: this class holds only the keys of the core linguistic layers * every pipeline shares (sentences, tokens, tags, entities). A capability-specific * layer's key lives on the annotator that provides it, for example the lemma layer's diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index 3d0ab766d3..9ccd37457b 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -122,6 +122,26 @@ void testDocumentScopedLayersCarrySpanlessValues() { assertEquals(2, both.layers().size()); } + /** + * Verifies the gold-layer convention: a hand-annotated version of a layer lives + * under the {@code gold:} prefixed key beside the produced layer, both are readable + * independently, and the once-only rule keeps either from replacing the other. + */ + @Test + void testGoldLayerLivesBesideThePredictedLayer() { + final LayerKey gold = LayerKey.of("gold:" + Layers.TOKENS.id(), String.class); + final Document document = Document.of("the dog") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "the"))) + .with(gold, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + assertEquals("gold:opennlp:tokens", gold.id()); + assertEquals(1, document.get(Layers.TOKENS).size()); + assertEquals(2, document.get(gold).size()); + assertThrows(IllegalArgumentException.class, + () -> document.with(gold, List.of())); + } + /** * Verifies that a layer preserves the insertion order of its annotations: the * container does not sort by span, so a producer that wants span order must supply diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index bf9b4b10f9..328163b5ae 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -79,6 +79,13 @@ LayerKey LANGUAGE = LayerKey.document("app:language", String.class); Document tagged = document.with(LANGUAGE, List.of(Annotation.of("eng"))); String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]> + + A corpus may carry a hand-annotated version of a layer beside a produced one. + The convention is a gold: id prefix on the same key scheme, for + example gold:opennlp:tokens beside opennlp:tokens; + since adding a layer is once-only, competing versions always live under + distinct keys and never replace each other. +

From 266e780d920e4567bbfdc03d9ee3d72b877a590d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 12:14:57 -0400 Subject: [PATCH 12/92] OPENNLP-1888: Create toolkit layer keys through namespace-applying factories on Layers --- .../java/opennlp/tools/document/Layers.java | 72 +++++++++++++++++-- .../tools/document/DocumentContractTest.java | 28 ++++++++ .../tools/lemmatizer/LemmatizerAnnotator.java | 2 +- .../tools/stemmer/StemmerAnnotator.java | 2 +- opennlp-docs/src/docbkx/document.xml | 3 +- 5 files changed, 99 insertions(+), 8 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index cec6960eb2..23322b158b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -17,6 +17,8 @@ package opennlp.tools.document; +import opennlp.tools.util.StringUtil; + /** * The standard {@link LayerKey layer keys} for the results the toolkit produces itself. * @@ -27,7 +29,9 @@ *

Namespace rule: every key the toolkit itself defines carries the * {@code opennlp:} id prefix. An extension defines its keys under its own prefix, and * a bare id without a prefix is legal for an application-local layer, so ids from - * independent producers cannot collide.

+ * independent producers cannot collide. Toolkit keys are created through + * {@link #key(String, Class)} and {@link #documentKey(String, Class)}, which apply the + * prefix, so no producer spells it.

* *

Gold versus predicted: a corpus may carry a hand-annotated version of a layer * beside a produced one. The convention is a {@code gold:} id prefix on the same key @@ -44,26 +48,84 @@ */ public final class Layers { + /** The id prefix of every key the toolkit defines. */ + private static final String NAMESPACE = "opennlp:"; + /** * Sentence boundaries; each annotation covers one sentence and carries its text. */ - public static final LayerKey SENTENCES = LayerKey.of("opennlp:sentences", String.class); + public static final LayerKey SENTENCES = key("sentences", String.class); /** * Token boundaries; each annotation covers one token and carries its text. */ - public static final LayerKey TOKENS = LayerKey.of("opennlp:tokens", String.class); + public static final LayerKey TOKENS = key("tokens", String.class); /** * Part-of-speech tags; one annotation per token, aligned with {@link #TOKENS} by * position, carrying the tag. */ - public static final LayerKey POS_TAGS = LayerKey.of("opennlp:pos", String.class); + public static final LayerKey POS_TAGS = key("pos", String.class); /** * Named entities; each annotation covers one mention and carries the entity type. */ - public static final LayerKey ENTITIES = LayerKey.of("opennlp:entities", String.class); + public static final LayerKey ENTITIES = key("entities", String.class); + + /** + * Creates a {@link LayerKey.Scope#POSITIONAL positional} key in the toolkit's + * {@code opennlp:} namespace, for a layer the toolkit itself produces. + * + * @param name The layer name without a namespace, for example {@code tokens}. Must + * not be {@code null}, blank, or contain {@code ':'}. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A key whose id is the name under the {@code opennlp:} prefix. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or + * contains {@code ':'}, or {@code type} is {@code null}. + */ + public static LayerKey key(String name, Class type) { + return LayerKey.of(NAMESPACE + validName(name), type); + } + + /** + * Creates a {@link LayerKey.Scope#DOCUMENT document-scoped} key in the toolkit's + * {@code opennlp:} namespace, for a whole-document value the toolkit itself produces. + * + * @param name The layer name without a namespace, for example {@code language}. Must + * not be {@code null}, blank, or contain {@code ':'}. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param The type of the annotation values. + * @return A document-scoped key whose id is the name under the {@code opennlp:} + * prefix. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or + * contains {@code ':'}, or {@code type} is {@code null}. + */ + public static LayerKey documentKey(String name, Class type) { + return LayerKey.document(NAMESPACE + validName(name), type); + } + + /** + * Validates a namespace-free layer name. + * + * @param name The name to validate. + * @return The validated name. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or + * contains {@code ':'}. + */ + private static String validName(String name) { + if (name == null || StringUtil.isBlank(name)) { + throw new IllegalArgumentException("name must not be null or blank"); + } + if (name.indexOf(':') >= 0) { + throw new IllegalArgumentException( + "name must not contain ':', the namespace is applied by this factory: " + name); + } + return name; + } private Layers() { // This class holds constants only and is never instantiated. diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index 9ccd37457b..fb01047aa5 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -122,6 +122,34 @@ void testDocumentScopedLayersCarrySpanlessValues() { assertEquals(2, both.layers().size()); } + /** + * Verifies the toolkit-namespace factories: {@link Layers#key(String, Class)} and + * {@link Layers#documentKey(String, Class)} apply the {@code opennlp:} prefix and + * yield keys equal to independently spelled ones, and a name that is null, blank, or + * already carries a namespace is rejected. + */ + @Test + void testToolkitKeysCarryTheNamespacePrefix() { + assertEquals(LayerKey.of("opennlp:things", String.class), + Layers.key("things", String.class)); + assertEquals("opennlp:things", Layers.key("things", String.class).id()); + assertEquals(Layers.TOKENS, Layers.key("tokens", String.class)); + + final LayerKey whole = Layers.documentKey("language", String.class); + assertEquals("opennlp:language", whole.id()); + assertEquals(LayerKey.Scope.DOCUMENT, whole.scope()); + + assertThrows(IllegalArgumentException.class, () -> Layers.key(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> Layers.key(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> Layers.key("things", null)); + final IllegalArgumentException nested = assertThrows(IllegalArgumentException.class, + () -> Layers.key("opennlp:things", String.class)); + assertEquals("name must not contain ':', the namespace is applied by this factory: " + + "opennlp:things", nested.getMessage()); + assertThrows(IllegalArgumentException.class, + () -> Layers.documentKey("app:x", String.class)); + } + /** * Verifies the gold-layer convention: a hand-annotated version of a layer lives * under the {@code gold:} prefixed key beside the produced layer, both are readable diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index c3f5897d6b..a72698e6f8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -46,7 +46,7 @@ public class LemmatizerAnnotator implements DocumentAnnotator { * The lemma layer. It is aligned with the token layer by position, and each annotation * carries the lemma of its token on that token's span. */ - public static final LayerKey LEMMAS = LayerKey.of("opennlp:lemmas", String.class); + public static final LayerKey LEMMAS = Layers.key("lemmas", String.class); /** The message prefix of every absent-required-layer rejection in this adapter. */ private static final String MISSING_LAYER = "document lacks the required layer "; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index 015779ee07..a8c00ba8f4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -42,7 +42,7 @@ public class StemmerAnnotator implements DocumentAnnotator { * The stem layer. It is aligned with the token layer by position, and each annotation * carries the stem of its token on that token's span. */ - public static final LayerKey STEMS = LayerKey.of("opennlp:stems", String.class); + public static final LayerKey STEMS = Layers.key("stems", String.class); private final Stemmer stemmer; diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 328163b5ae..6e2f2d018c 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -51,7 +51,8 @@ are namespaced: keys the toolkit defines carry the opennlp: prefix, an extension uses its own prefix, and a bare id, like the token-lengths key below, is legal for an application-local - layer. + layer. Toolkit keys are created through Layers.key and + Layers.documentKey, which apply the prefix. From d43c1ec9ada7017df49b64b150039d061f5e08bb Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:38:12 -0400 Subject: [PATCH 13/92] OPENNLP-1888: Javadoc container overrides and cite the pipeline example test Add {@inheritDoc} to the Document, LayerKey, and adapter overrides, and note in the manual that DocumentPipelineExampleTest asserts the pipeline round-trip. --- .../main/java/opennlp/tools/document/ImmutableDocument.java | 4 ++++ .../src/main/java/opennlp/tools/document/LayerKey.java | 3 +++ .../main/java/opennlp/tools/document/NameFinderAnnotator.java | 2 ++ .../main/java/opennlp/tools/document/POSTaggerAnnotator.java | 2 ++ .../opennlp/tools/document/SentenceDetectorAnnotator.java | 1 + .../main/java/opennlp/tools/document/TokenizerAnnotator.java | 1 + opennlp-docs/src/docbkx/document.xml | 4 ++++ 7 files changed, 17 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index e2ca8cd5f3..bcad31ec46 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -54,11 +54,13 @@ static ImmutableDocument empty(CharSequence text) { return new ImmutableDocument(text, Collections.emptyMap()); } + /** {@inheritDoc} */ @Override public CharSequence text() { return text; } + /** {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public List> get(LayerKey layer) { @@ -74,11 +76,13 @@ public List> get(LayerKey layer) { return (List>) (List) annotations; } + /** {@inheritDoc} */ @Override public Set> layers() { return Collections.unmodifiableSet(layers.keySet()); } + /** {@inheritDoc} */ @Override public Document with(LayerKey layer, List> annotations) { if (layer == null || annotations == null) { diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java index 7d72411bd5..e29870a06c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -139,6 +139,7 @@ public Scope scope() { return scope; } + /** {@inheritDoc} */ @Override public boolean equals(Object obj) { if (this == obj) { @@ -150,11 +151,13 @@ public boolean equals(Object obj) { return id.equals(other.id) && type.equals(other.type) && scope == other.scope; } + /** {@inheritDoc} */ @Override public int hashCode() { return Objects.hash(id, type, scope); } + /** {@inheritDoc} */ @Override public String toString() { return id + '<' + type.getSimpleName() + '>'; diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java index 75c33496c8..8b1d6e3e7a 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -153,11 +153,13 @@ public Document annotate(Document document) { return document.with(Layers.ENTITIES, entities); } + /** {@inheritDoc} */ @Override public Set> requires() { return Set.of(Layers.SENTENCES, Layers.TOKENS); } + /** {@inheritDoc} */ @Override public Set> provides() { return Set.of(Layers.ENTITIES); diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java index 08e1b81f8c..520c99cf0e 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -125,11 +125,13 @@ public Document annotate(Document document) { return document.with(Layers.POS_TAGS, tagAnnotations); } + /** {@inheritDoc} */ @Override public Set> requires() { return Set.of(Layers.SENTENCES, Layers.TOKENS); } + /** {@inheritDoc} */ @Override public Set> provides() { return Set.of(Layers.POS_TAGS); diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java index 282edf31dd..136987401e 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java @@ -73,6 +73,7 @@ public Document annotate(Document document) { return document.with(Layers.SENTENCES, sentences); } + /** {@inheritDoc} */ @Override public Set> provides() { return Set.of(Layers.SENTENCES); diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java index e9384bd7b9..04ddccc591 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -95,6 +95,7 @@ private void addTokens(List> tokens, String text, int offset) } } + /** {@inheritDoc} */ @Override public Set> provides() { return Set.of(Layers.TOKENS); diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 6e2f2d018c..29a009a884 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -122,6 +122,10 @@ DocumentAnalyzer analyzer = DocumentAnalyzer.builder() Document document = analyzer.analyze("The dog barks. It naps.");]]> + + DocumentPipelineExampleTest asserts the pipeline and layer + round-trip shown here. + The resulting document carries exactly the four layers the pipeline provides. The sentence layer holds two annotations, [0..14) covering From 04532170cb2cad7351457e3cc3a9158d38306e81 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 05:27:41 -0400 Subject: [PATCH 14/92] OPENNLP-1888: Anchor the Document thread-safety note to immutability --- opennlp-api/src/main/java/opennlp/tools/document/Document.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index dc4be9616c..7b3a3989f3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -33,7 +33,7 @@ * without spans, for example a language id.

* *

Documents are immutable: {@link #with(LayerKey, List)} returns a new document that - * shares the unchanged layers. Instances are safe to share between threads.

+ * shares the unchanged layers. That immutability makes instances safe to share between threads.

* *

Three invariants make index-based references sound. A layer preserves its * insertion order, and the container never sorts or reorders it. A layer is immutable From 15102d75dddfdaa4e6394438782c8b7834a46de5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 02:41:27 -0400 Subject: [PATCH 15/92] OPENNLP-1888: Address review: shared annotator helpers, hardened container contract - Reject zero-length finder mentions in NameFinderAnnotator and pin the second-sentence case, which was previously mapped silently wrong, with a test - Add DocumentAnnotators with requireLayers and the per-sentence token walk, replacing three copies of the walk loop and four spellings of the missing-layer rejection; direct tests pin the helpers as public API - Capture the document text as a String at construction so ImmutableDocument's immutability and thread-safety claims hold for mutable CharSequence inputs - Move the copy-on-add and threading narrative from the Document interface Javadoc to ImmutableDocument; the interface now states that thread safety is implementation specific - Carry the entity type as the annotation value only; entity spans are untyped, and the Javadoc names the value as the single source of the type - Make all six adapter annotators final before the types freeze - Align the TokenLengthAnnotator example with the documented required-layer contract in both the manual and the example test via requireLayers - Housekeeping per review: docbook CDATA placement, imports over qualified names, a ParameterizedTest for the blank-input matrix, shared deterministic test components, static assertion imports, inheritDoc on the runtime adapters, Layers constructor comment, and the documented NPE of StringUtil.isBlank --- .../java/opennlp/tools/document/Document.java | 13 +- .../tools/document/DocumentAnnotators.java | 118 +++++++++++++++++ .../tools/document/ImmutableDocument.java | 19 ++- .../java/opennlp/tools/document/Layers.java | 5 +- .../tools/document/NameFinderAnnotator.java | 73 +++-------- .../tools/document/POSTaggerAnnotator.java | 49 +------ .../document/SentenceDetectorAnnotator.java | 2 +- .../tools/document/TokenizerAnnotator.java | 2 +- .../java/opennlp/tools/util/StringUtil.java | 1 + .../tools/document/DocumentAnalyzerTest.java | 98 ++++---------- .../document/DocumentAnnotatorsTest.java | 121 ++++++++++++++++++ .../document/DocumentPipelineExampleTest.java | 83 ++---------- .../opennlp/tools/document/DocumentTest.java | 17 +++ .../document/NameFinderAnnotatorTest.java | 48 ++++++- .../tools/document/TestComponents.java | 92 +++++++++++++ .../tools/lemmatizer/LemmatizerAnnotator.java | 60 ++------- .../tools/stemmer/StemmerAnnotator.java | 13 +- .../lemmatizer/LemmatizerAnnotatorTest.java | 51 ++++---- .../tools/stemmer/StemmerAnnotatorTest.java | 25 ++-- opennlp-docs/src/docbkx/document.xml | 27 ++-- 20 files changed, 553 insertions(+), 364 deletions(-) create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/document/TestComponents.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 7b3a3989f3..4169793eff 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -21,8 +21,8 @@ import java.util.Set; /** - * An immutable, offset-anchored annotation container: the original text of one document - * plus any number of typed annotation layers over it. + * An offset-anchored annotation container: the original text of one document plus any + * number of typed annotation layers over it. * *

A layer is a list of {@link Annotation annotations} identified by a * {@link LayerKey}. The container itself knows nothing about specific layers; every @@ -32,8 +32,9 @@ * {@link LayerKey.Scope#DOCUMENT document-scoped} layer carries whole-document values * without spans, for example a language id.

* - *

Documents are immutable: {@link #with(LayerKey, List)} returns a new document that - * shares the unchanged layers. That immutability makes instances safe to share between threads.

+ *

A document is never modified in place: {@link #with(LayerKey, List)} leaves its + * receiver untouched and returns a new document. Thread safety is implementation + * specific.

* *

Three invariants make index-based references sound. A layer preserves its * insertion order, and the container never sorts or reorders it. A layer is immutable @@ -49,7 +50,9 @@ public interface Document { /** - * Creates an empty {@link Document} over a text. + * Creates an empty {@link Document} over a text. The returned document is immutable + * and safe to share between threads: it captures the text's content at construction, + * so later changes to a mutable {@code CharSequence} do not reach the document. * * @param text The original document text. Must not be {@code null}. * @return A {@link Document} without any layers. Never {@code null}. diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java new file mode 100644 index 0000000000..fd5f81b858 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +/** + * Support methods shared by {@link DocumentAnnotator} implementations: the + * required-layer check and the per-sentence walk over the token layer. + * + *

These helpers keep the annotators' shared behavior identical across + * implementations: an absent required layer is always rejected with the same message + * naming the layer, and every per-sentence adapter applies the same sentence-to-token + * mapping, including the loud rejection of a token lying outside every sentence.

+ * + * @since 3.0.0 + */ +public final class DocumentAnnotators { + + /** + * Receives one sentence's contiguous token run during + * {@link #forEachSentence(List, List, SentenceTokenConsumer)}. + */ + @FunctionalInterface + public interface SentenceTokenConsumer { + + /** + * Consumes one sentence's tokens. + * + * @param first The position of the sentence's first token in the token layer. + * @param words The sentence's token values in layer order. Never {@code null} or + * empty; the run covers the token layer positions + * {@code [first, first + words.length)}. + */ + void accept(int first, String[] words); + } + + /** + * Verifies that a document is present and carries every given layer. + * + * @param document The document to check. + * @param layers The required layers, in the order they are to be reported. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, or if + * a layer is absent; the message names the first absent layer. + */ + public static void requireLayers(Document document, LayerKey... layers) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final Set> present = document.layers(); + for (final LayerKey layer : layers) { + if (!present.contains(layer)) { + throw new IllegalArgumentException("document lacks the required layer " + layer); + } + } + } + + /** + * Walks the token layer sentence by sentence and hands each sentence's contiguous + * token run to the consumer. + * + *

Both layers must be in text order. Each sentence consumes the contiguous run of + * tokens whose spans it encloses; a sentence without tokens is skipped. Every token + * must belong to a sentence: a token lying outside every sentence is rejected loudly + * after the walk, so it can never be silently dropped.

+ * + * @param sentences The sentence layer, in text order. Must not be {@code null}. + * @param tokens The token layer, in text order. Must not be {@code null}. + * @param consumer Receives each token-carrying sentence's run. Must not be + * {@code null}. + * @throws IllegalArgumentException Thrown if a token lies outside every sentence. + */ + public static void forEachSentence(List> sentences, + List> tokens, SentenceTokenConsumer consumer) { + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + final String[] words = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + } + consumer.accept(first, words); + } + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); + } + } + + private DocumentAnnotators() { + // Not instantiated; this class provides static support methods only. + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index bcad31ec46..b456258d3d 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -27,15 +27,21 @@ /** * The default {@link Document} implementation: an unmodifiable map from layer key to an - * unmodifiable annotation list. Adding a layer copies the map, not the layers, so - * documents grown from a common ancestor share their layer lists. + * unmodifiable annotation list. + * + *

Instances are immutable: {@link #with(LayerKey, List)} returns a new document that + * shares the unchanged layers with its ancestor, copying the map but not the layers, so + * documents grown from a common ancestor share their layer lists. The text is captured + * as a {@link String} at construction, so a mutable {@link CharSequence} handed to + * {@link #empty(CharSequence)} cannot change the document afterwards. That immutability + * makes instances safe to share between threads.

*/ final class ImmutableDocument implements Document { - private final CharSequence text; + private final String text; private final Map, List>> layers; - private ImmutableDocument(CharSequence text, Map, List>> layers) { + private ImmutableDocument(String text, Map, List>> layers) { this.text = text; this.layers = layers; } @@ -43,7 +49,8 @@ private ImmutableDocument(CharSequence text, Map, List /** * Creates a document without any layers. * - * @param text The original document text. Must not be {@code null}. + * @param text The original document text, captured as its content at this moment. + * Must not be {@code null}. * @return An empty {@link ImmutableDocument}. Never {@code null}. * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ @@ -51,7 +58,7 @@ static ImmutableDocument empty(CharSequence text) { if (text == null) { throw new IllegalArgumentException("text must not be null"); } - return new ImmutableDocument(text, Collections.emptyMap()); + return new ImmutableDocument(text.toString(), Collections.emptyMap()); } /** {@inheritDoc} */ diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index 23322b158b..0257afc384 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -68,7 +68,8 @@ public final class Layers { public static final LayerKey POS_TAGS = key("pos", String.class); /** - * Named entities; each annotation covers one mention and carries the entity type. + * Named entities; each annotation covers one mention and carries the entity type as + * its value. The annotation's span carries offsets only. */ public static final LayerKey ENTITIES = key("entities", String.class); @@ -128,6 +129,6 @@ private static String validName(String name) { } private Layers() { - // This class holds constants only and is never instantiated. + // Not instantiated; this class provides constants and static key factories only. } } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java index 8b1d6e3e7a..677e0dfa20 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -28,8 +28,9 @@ /** * Adapts a {@link TokenNameFinder} to the document pipeline: reads * {@link Layers#SENTENCES} and {@link Layers#TOKENS}, maps the finder's token-index - * spans to character spans on the original text, and provides {@link Layers#ENTITIES} - * carrying the entity type. + * spans to character spans on the original text, and provides {@link Layers#ENTITIES}. + * The entity type is carried as the annotation value; the annotation's span carries + * offsets only. * *

Each sentence's tokens are passed to {@link TokenNameFinder#find(String[])} as one * sequence, the way the finder contract expects its input, so no mention can straddle a @@ -42,7 +43,7 @@ * * @since 3.0.0 */ -public class NameFinderAnnotator implements DocumentAnnotator { +public final class NameFinderAnnotator implements DocumentAnnotator { /** * The entity type recorded when the wrapped finder returns a span without a type. It @@ -52,9 +53,6 @@ public class NameFinderAnnotator implements DocumentAnnotator { */ public static final String UNTYPED = NameSample.DEFAULT_TYPE; - /** The message prefix of every absent-required-layer rejection in this adapter. */ - private static final String MISSING_LAYER = "document lacks the required layer "; - private final TokenNameFinder finder; /** @@ -79,7 +77,7 @@ public NameFinderAnnotator(TokenNameFinder finder) { * text. The required layers must be present, but they may be empty: a document * without sentences or tokens yields a present-but-empty entity layer, and a sentence * containing no tokens contributes nothing. A mention without a type is recorded with - * the type {@link #UNTYPED}.

+ * the type {@link #UNTYPED} as the annotation value.

* * @param document The document to annotate. Must not be {@code null} and must carry * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with @@ -88,65 +86,36 @@ public NameFinderAnnotator(TokenNameFinder finder) { * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the * sentence layer or the token layer is absent, a token lies outside every - * sentence, or the finder returns a mention whose token indices lie outside - * its sentence's tokens. + * sentence, or the finder returns a mention that is empty or whose token + * indices lie outside its sentence's tokens. */ @Override public Document annotate(Document document) { - if (document == null) { - throw new IllegalArgumentException("document must not be null"); - } - final Set> present = document.layers(); - if (!present.contains(Layers.SENTENCES)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.SENTENCES); - } - if (!present.contains(Layers.TOKENS)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.TOKENS); - } + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); final List> sentences = document.get(Layers.SENTENCES); final List> tokens = document.get(Layers.TOKENS); final List> entities = new ArrayList<>(); - // Walk the token layer once: both layers are in text order, so each sentence - // consumes the contiguous run of tokens whose spans it encloses. The adaptive data - // is cleared even when annotation fails, so a rejected document cannot leak finder - // state into the next one. + // The adaptive data is cleared even when annotation fails, so a rejected document + // cannot leak finder state into the next one. try { - int next = 0; - for (final Annotation sentence : sentences) { - final int first = next; - while (next < tokens.size() - && tokens.get(next).span().getStart() >= sentence.span().getStart() - && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { - next++; - } - final int count = next - first; - if (count == 0) { - continue; - } - final String[] words = new String[count]; - for (int i = 0; i < count; i++) { - words[i] = tokens.get(first + i).value(); - } - // The finder indexes within the sentence; shifting by the sentence's first token - // position turns every mention boundary into a document-wide token index, whose - // token spans already refer to the original text. + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + // The finder indexes within the sentence; shifting by the sentence's first + // token position turns every mention boundary into a document-wide token + // index, whose token spans already refer to the original text. An empty + // mention is rejected with the out-of-bounds ones: it covers no token, so it + // has no character span. for (final Span mention : finder.find(words)) { - if (mention.getStart() < 0 || mention.getEnd() > count) { + if (mention.getStart() < 0 || mention.getEnd() > words.length + || mention.getStart() >= mention.getEnd()) { throw new IllegalArgumentException("finder returned mention " + mention - + " outside the sentence's " + count + " tokens"); + + " outside the sentence's " + words.length + " tokens"); } final int start = tokens.get(first + mention.getStart()).span().getStart(); final int end = tokens.get(first + mention.getEnd() - 1).span().getEnd(); final String type = mention.getType() == null ? UNTYPED : mention.getType(); - entities.add(new Annotation<>(new Span(start, end, type), type)); + entities.add(new Annotation<>(new Span(start, end), type)); } - } - if (next != tokens.size()) { - throw new IllegalArgumentException("token at " + tokens.get(next).span() - + " lies outside every sentence"); - } + }); } finally { finder.clearAdaptiveData(); } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java index 520c99cf0e..b97aa98b4b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -36,10 +36,7 @@ * * @since 3.0.0 */ -public class POSTaggerAnnotator implements DocumentAnnotator { - - /** The message prefix of every absent-required-layer rejection in this adapter. */ - private static final String MISSING_LAYER = "document lacks the required layer "; +public final class POSTaggerAnnotator implements DocumentAnnotator { private final POSTagger tagger; @@ -76,52 +73,20 @@ public POSTaggerAnnotator(POSTagger tagger) { */ @Override public Document annotate(Document document) { - if (document == null) { - throw new IllegalArgumentException("document must not be null"); - } - final Set> present = document.layers(); - if (!present.contains(Layers.SENTENCES)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.SENTENCES); - } - if (!present.contains(Layers.TOKENS)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.TOKENS); - } + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); final List> sentences = document.get(Layers.SENTENCES); final List> tokens = document.get(Layers.TOKENS); final List> tagAnnotations = new ArrayList<>(tokens.size()); - // Walk the token layer once: both layers are in text order, so each sentence - // consumes the contiguous run of tokens whose spans it encloses. - int next = 0; - for (final Annotation sentence : sentences) { - final int first = next; - while (next < tokens.size() - && tokens.get(next).span().getStart() >= sentence.span().getStart() - && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { - next++; - } - final int count = next - first; - if (count == 0) { - continue; - } - final String[] words = new String[count]; - for (int i = 0; i < count; i++) { - words[i] = tokens.get(first + i).value(); - } + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { final String[] tags = tagger.tag(words); - if (tags.length != count) { + if (tags.length != words.length) { throw new IllegalArgumentException( - "tagger returned " + tags.length + " tags for " + count + " tokens"); + "tagger returned " + tags.length + " tags for " + words.length + " tokens"); } - for (int i = 0; i < count; i++) { + for (int i = 0; i < words.length; i++) { tagAnnotations.add(new Annotation<>(tokens.get(first + i).span(), tags[i])); } - } - if (next != tokens.size()) { - throw new IllegalArgumentException("token at " + tokens.get(next).span() - + " lies outside every sentence"); - } + }); return document.with(Layers.POS_TAGS, tagAnnotations); } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java index 136987401e..a035f0517e 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java @@ -33,7 +33,7 @@ * * @since 3.0.0 */ -public class SentenceDetectorAnnotator implements DocumentAnnotator { +public final class SentenceDetectorAnnotator implements DocumentAnnotator { private final SentenceDetector detector; diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java index 04ddccc591..6a81fcea9c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -35,7 +35,7 @@ * * @since 3.0.0 */ -public class TokenizerAnnotator implements DocumentAnnotator { +public final class TokenizerAnnotator implements DocumentAnnotator { private final Tokenizer tokenizer; diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 1842556720..89694335cc 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -277,6 +277,7 @@ public static boolean isEmpty(CharSequence theString) { * * @param theString The {@link CharSequence} to examine. Must not be {@code null}. * @return {@code true} if {@code theString} is empty or all whitespace. + * @throws NullPointerException Thrown if {@code theString} is {@code null}. */ public static boolean isBlank(CharSequence theString) { for (int i = 0; i < theString.length(); ) { diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java index deb317640f..7b17c63692 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -21,11 +21,12 @@ import java.util.Set; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.namefind.TokenNameFinder; import opennlp.tools.postag.POSTagger; -import opennlp.tools.sentdetect.SentenceDetector; -import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Sequence; import opennlp.tools.util.Span; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,67 +34,22 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using simple - * inline implementations of the task interfaces: whitespace tokenization, period sentence - * splitting, and a dictionary tagger. The point under test is the pipeline mechanics and - * span arithmetic, not model quality. + * Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using the + * deterministic components from {@link TestComponents} and a fixed-vocabulary tagger. + * The point under test is the pipeline mechanics and span arithmetic, not model quality. */ public class DocumentAnalyzerTest { - private static final SentenceDetector SPLITTER = new SentenceDetector() { - - @Override - public String[] sentDetect(CharSequence s) { - throw new UnsupportedOperationException(); - } - - @Override - public Span[] sentPosDetect(CharSequence s) { - // split after each period; keep it simple for the test - final String text = s.toString(); - int start = 0; - final java.util.List spans = new java.util.ArrayList<>(); - for (int i = 0; i < text.length(); i++) { - if (text.charAt(i) == '.') { - spans.add(new Span(start, i + 1)); - start = i + 2; - } - } - return spans.toArray(new Span[0]); - } - }; - - private static final Tokenizer WHITESPACE = new Tokenizer() { - - @Override - public String[] tokenize(String s) { - throw new UnsupportedOperationException(); - } - - @Override - public Span[] tokenizePos(String s) { - final java.util.List spans = new java.util.ArrayList<>(); - int start = -1; - for (int i = 0; i <= s.length(); i++) { - final boolean boundary = i == s.length() || s.charAt(i) == ' '; - if (boundary && start >= 0) { - spans.add(new Span(start, i)); - start = -1; - } else if (!boundary && start < 0) { - start = i; - } - } - return spans.toArray(new Span[0]); - } - }; - + /** Tags the known verbs of the test texts {@code VBZ} and everything else {@code X}. */ private static final POSTagger TAGGER = new POSTagger() { + private final Set verbs = Set.of("barks.", "eats."); + @Override public String[] tag(String[] sentence) { final String[] tags = new String[sentence.length]; for (int i = 0; i < sentence.length; i++) { - tags[i] = "barks.".contains(sentence[i]) ? "VBZ" : "X"; + tags[i] = verbs.contains(sentence[i]) ? "VBZ" : "X"; } return tags; } @@ -104,13 +60,12 @@ public String[] tag(String[] sentence, Object[] additionalContext) { } @Override - public opennlp.tools.util.Sequence[] topKSequences(String[] sentence) { + public Sequence[] topKSequences(String[] sentence) { throw new UnsupportedOperationException(); } @Override - public opennlp.tools.util.Sequence[] topKSequences(String[] sentence, - Object[] additionalContext) { + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { throw new UnsupportedOperationException(); } }; @@ -118,8 +73,8 @@ public opennlp.tools.util.Sequence[] topKSequences(String[] sentence, @Test void testPipelineProducesAlignedLayersInOriginalCoordinates() { final Document document = DocumentAnalyzer.builder() - .add(new SentenceDetectorAnnotator(SPLITTER)) - .add(new TokenizerAnnotator(WHITESPACE)) + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) .add(new POSTaggerAnnotator(TAGGER)) .build() .analyze("the dog barks. she eats."); @@ -145,8 +100,9 @@ void testPipelineProducesAlignedLayersInOriginalCoordinates() { * document on which every provided layer is present and empty, rather than failing: * zero sentences legitimately yield zero tokens, zero tags, and zero entities. */ - @Test - void testEmptyAndBlankInputProduceEmptyLayers() { + @ParameterizedTest + @ValueSource(strings = {"", " "}) + void testEmptyAndBlankInputProduceEmptyLayers(String text) { final TokenNameFinder finder = new TokenNameFinder() { @Override @@ -159,19 +115,17 @@ public void clearAdaptiveData() { } }; final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() - .add(new SentenceDetectorAnnotator(SPLITTER)) - .add(new TokenizerAnnotator(WHITESPACE)) + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) .add(new POSTaggerAnnotator(TAGGER)) .add(new NameFinderAnnotator(finder)) .build(); - for (final String text : new String[] {"", " "}) { - final Document document = analyzer.analyze(text); - assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, Layers.ENTITIES), - document.layers()); - for (final LayerKey layer : document.layers()) { - assertTrue(document.get(layer).isEmpty()); - } + final Document document = analyzer.analyze(text); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, Layers.ENTITIES), + document.layers()); + for (final LayerKey layer : document.layers()) { + assertTrue(document.get(layer).isEmpty()); } } @@ -181,7 +135,7 @@ public void clearAdaptiveData() { */ @Test void testTokenizerHonorsPresentButEmptySentenceLayer() { - final Document document = new TokenizerAnnotator(WHITESPACE) + final Document document = new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER) .annotate(Document.of("the dog").with(Layers.SENTENCES, List.of())); assertTrue(document.layers().contains(Layers.TOKENS)); assertTrue(document.get(Layers.TOKENS).isEmpty()); @@ -190,7 +144,7 @@ void testTokenizerHonorsPresentButEmptySentenceLayer() { @Test void testTokenizerWorksWithoutSentences() { final Document document = DocumentAnalyzer.builder() - .add(new TokenizerAnnotator(WHITESPACE)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) .build() .analyze("the dog"); assertEquals(2, document.get(Layers.TOKENS).size()); diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java new file mode 100644 index 0000000000..bef5db118f --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link DocumentAnnotators} support methods directly: the required-layer + * check's exact rejection messages and the per-sentence walk's slicing, skipping, and + * loud rejection of a token outside every sentence. The adapter tests exercise the same + * behavior through the annotators; this class pins the helpers as public API on their + * own. + */ +public class DocumentAnnotatorsTest { + + @Test + void testRequireLayersAcceptsPresentLayers() { + final Document document = Document.of("the") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()); + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); + DocumentAnnotators.requireLayers(document); + } + + @Test + void testRequireLayersRejectsNullDocument() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnnotators.requireLayers(null, Layers.TOKENS)); + assertEquals("document must not be null", e.getMessage()); + } + + /** + * Verifies that an absent layer is rejected with the shared message naming the first + * absent layer in the order the caller listed them. + */ + @Test + void testRequireLayersNamesTheFirstAbsentLayer() { + final Document document = Document.of("the").with(Layers.TOKENS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnnotators.requireLayers(document, + Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS)); + assertEquals("document lacks the required layer opennlp:sentences", + e.getMessage()); + } + + /** + * Verifies the walk contract: each sentence receives the contiguous run of tokens its + * span encloses with the run's first token layer position, and a sentence without + * tokens is skipped rather than reported as an empty run. + */ + @Test + void testForEachSentenceSlicesContiguousRuns() { + final List> sentences = List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 11), "!"), + new Annotation<>(new Span(12, 21), "Bob sits.")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(12, 15), "Bob"), + new Annotation<>(new Span(16, 21), "sits.")); + + final List firsts = new ArrayList<>(); + final List> runs = new ArrayList<>(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + firsts.add(first); + runs.add(List.of(words)); + }); + + assertEquals(List.of(0, 2), firsts); + assertEquals(List.of( + List.of("Ana", "runs."), + List.of("Bob", "sits.")), runs); + } + + @Test + void testForEachSentenceRejectsTokenOutsideEverySentence() { + final List> sentences = List.of( + new Annotation<>(new Span(0, 9), "Ana runs.")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob")); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + })); + assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + } + + @Test + void testForEachSentenceOverEmptyLayersConsumesNothing() { + final List> runs = new ArrayList<>(); + DocumentAnnotators.forEachSentence(List.of(), List.of(), + (first, words) -> runs.add(List.of(words))); + assertTrue(runs.isEmpty()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java index 14eeda39a4..0595b53716 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java @@ -25,8 +25,6 @@ import org.junit.jupiter.api.Test; import opennlp.tools.postag.POSTagger; -import opennlp.tools.sentdetect.SentenceDetector; -import opennlp.tools.tokenize.Tokenizer; import opennlp.tools.util.Sequence; import opennlp.tools.util.Span; @@ -39,9 +37,10 @@ * {@link DocumentAnalyzer}, analyze a two-sentence text, and read every layer back with * its spans in original text coordinates. * - *

The wrapped components are tiny deterministic stand-ins defined in this class, so - * every expected span and value below follows directly from the input text. The point - * under demonstration is how the layers connect, not the quality of any single step.

+ *

The wrapped components are the tiny deterministic stand-ins from + * {@link TestComponents}, so every expected span and value below follows directly from + * the input text. The point under demonstration is how the layers connect, not the + * quality of any single step.

*/ public class DocumentPipelineExampleTest { @@ -52,62 +51,6 @@ public class DocumentPipelineExampleTest { private static final LayerKey TOKEN_LENGTHS = LayerKey.of("token-lengths", Integer.class); - /** - * A deterministic sentence detector that ends a sentence after every period and - * expects a single space between sentences. Only the span-producing method is - * implemented because the adapter calls no other method. - */ - private static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { - - @Override - public String[] sentDetect(CharSequence s) { - throw new UnsupportedOperationException("the adapter only calls sentPosDetect"); - } - - @Override - public Span[] sentPosDetect(CharSequence s) { - final String text = s.toString(); - final List spans = new ArrayList<>(); - int start = 0; - for (int i = 0; i < text.length(); i++) { - if (text.charAt(i) == '.') { - spans.add(new Span(start, i + 1)); - start = i + 2; - } - } - return spans.toArray(new Span[0]); - } - }; - - /** - * A deterministic tokenizer that splits on single space characters and keeps all - * other characters, including sentence-final periods, attached to their token. Only - * the span-producing method is implemented because the adapter calls no other method. - */ - private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { - - @Override - public String[] tokenize(String s) { - throw new UnsupportedOperationException("the adapter only calls tokenizePos"); - } - - @Override - public Span[] tokenizePos(String s) { - final List spans = new ArrayList<>(); - int start = -1; - for (int i = 0; i <= s.length(); i++) { - final boolean boundary = i == s.length() || s.charAt(i) == ' '; - if (boundary && start >= 0) { - spans.add(new Span(start, i)); - start = -1; - } else if (!boundary && start < 0) { - start = i; - } - } - return spans.toArray(new Span[0]); - } - }; - /** * A deterministic tagger backed by a fixed dictionary covering exactly the tokens of * the example text. An unknown token fails the test immediately rather than receiving @@ -158,21 +101,15 @@ private static final class TokenLengthAnnotator implements DocumentAnnotator { * Adds the {@link #TOKEN_LENGTHS} layer computed from {@link Layers#TOKENS}. * * @param document The document to annotate. Must not be {@code null} and must - * contain the token layer. + * contain the token layer, which may be empty. * @return A new {@link Document} carrying the token length layer. Never {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or the * token layer is absent. */ @Override public Document annotate(Document document) { - if (document == null) { - throw new IllegalArgumentException("document must not be null"); - } + DocumentAnnotators.requireLayers(document, Layers.TOKENS); final List> tokens = document.get(Layers.TOKENS); - if (tokens.isEmpty()) { - throw new IllegalArgumentException("document lacks the required layer " - + Layers.TOKENS); - } final List> lengths = new ArrayList<>(tokens.size()); for (final Annotation token : tokens) { lengths.add(new Annotation<>(token.span(), token.value().length())); @@ -214,8 +151,8 @@ void testDocumentScopedLayerExample() { @Test void testFullPipelineStory() { final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() - .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) - .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) .add(new POSTaggerAnnotator(DICTIONARY_TAGGER)) .add(new TokenLengthAnnotator()) .build(); @@ -289,8 +226,8 @@ void testFullPipelineStory() { @Test void testAnalyzerIsReusableAcrossTexts() { final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() - .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) - .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) .build(); final Document first = analyzer.analyze("The dog barks."); diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java index 846007652f..787c9d8edb 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java @@ -47,6 +47,23 @@ void testEmptyDocument() { assertTrue(document.get(WORDS).isEmpty()); } + /** + * Verifies that the text is captured at construction: mutating a + * {@link StringBuilder} handed to {@link Document#of(CharSequence)} does not reach + * the document, so the span bounds validated on insertion stay valid for its + * lifetime. + */ + @Test + void testTextIsCapturedAtConstruction() { + final StringBuilder mutable = new StringBuilder("the dog"); + final Document document = Document.of(mutable) + .with(WORDS, List.of(new Annotation<>(new Span(4, 7), "dog"))); + mutable.setLength(0); + assertEquals("the dog", document.text().toString()); + assertEquals("dog", + document.get(WORDS).get(0).span().getCoveredText(document.text()).toString()); + } + @Test void testWithAddsATypedLayer() { final Document document = Document.of("the dog") diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java index 99f80dae03..3b3e87d5bb 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -30,12 +30,14 @@ import opennlp.tools.util.Span; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests that {@link NameFinderAnnotator} maps token-index mentions to character spans on - * the original text and clears the finder's adaptive data per document. + * the original text, carries the entity type as the annotation value on an untyped span, + * and clears the finder's adaptive data per document. */ public class NameFinderAnnotatorTest { @@ -85,8 +87,10 @@ void testTokenIndexSpansBecomeCharacterSpans() { final Document annotated = new NameFinderAnnotator(finder).annotate(document); final List> entities = annotated.get(Layers.ENTITIES); assertEquals(1, entities.size()); - assertEquals(new Span(3, 11, "location"), entities.get(0).span()); + assertEquals(new Span(3, 11), entities.get(0).span()); + // the annotation value is the single source of the entity type; the span is untyped assertEquals("location", entities.get(0).value()); + assertNull(entities.get(0).span().getType()); assertEquals("New York", entities.get(0).span().getCoveredText(annotated.text()).toString()); assertEquals(1, cleared.get()); @@ -126,8 +130,8 @@ void testUntypedMentionRecordedAsUntyped() { assertEquals(1, entities.size()); assertEquals(NameSample.DEFAULT_TYPE, NameFinderAnnotator.UNTYPED); assertEquals(NameFinderAnnotator.UNTYPED, entities.get(0).value()); - assertEquals(new Span(0, 3, NameFinderAnnotator.UNTYPED), entities.get(0).span()); - assertEquals(NameFinderAnnotator.UNTYPED, entities.get(0).span().getType()); + assertEquals(new Span(0, 3), entities.get(0).span()); + assertNull(entities.get(0).span().getType()); } /** @@ -159,6 +163,36 @@ void testMentionOutsideSentenceTokensFailsLoud() { assertEquals(1, cleared.get()); } + /** + * Verifies that a zero-length mention is rejected loudly. {@link Span} permits + * {@code start == end}, but such a mention covers no token, so mapping its end + * through {@code end - 1} would read the previous sentence's last token instead of + * failing. The finder here returns the empty mention for the second sentence, the + * case that would otherwise be mapped silently wrong, and the adaptive data is still + * cleared on the failure. + */ + @Test + void testZeroLengthMentionFailsLoud() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = finder(tokens -> + "Bob".equals(tokens[0]) ? new Span[] {new Span(0, 0)} : new Span[0], cleared); + final Document document = Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(document)); + assertEquals("finder returned mention [0..0) outside the sentence's 2 tokens", + e.getMessage()); + assertEquals(1, cleared.get()); + } + /** * Verifies that a token lying outside every sentence is rejected loudly and that the * adaptive data is still cleared on that failure, so a rejected document cannot leak @@ -215,8 +249,10 @@ void testFindsPerSentenceAndMapsSentenceLocalIndices() { final List> entities = annotated.get(Layers.ENTITIES); assertEquals(2, entities.size()); - assertEquals(new Span(0, 3, "person"), entities.get(0).span()); - assertEquals(new Span(10, 13, "person"), entities.get(1).span()); + assertEquals(new Span(0, 3), entities.get(0).span()); + assertEquals("person", entities.get(0).value()); + assertEquals(new Span(10, 13), entities.get(1).span()); + assertEquals("person", entities.get(1).value()); assertEquals("Bob", entities.get(1).span().getCoveredText(annotated.text()).toString()); assertEquals(1, cleared.get()); diff --git a/opennlp-api/src/test/java/opennlp/tools/document/TestComponents.java b/opennlp-api/src/test/java/opennlp/tools/document/TestComponents.java new file mode 100644 index 0000000000..6145e5b790 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/TestComponents.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +/** + * Deterministic stand-in components shared by the document pipeline tests, so every + * expected span in those tests follows directly from the definitions here. + */ +final class TestComponents { + + /** + * A deterministic sentence detector that ends a sentence after every period and + * expects a single space between sentences. Only the span-producing method is + * implemented because the adapter calls no other method. + */ + static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the adapter only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A deterministic tokenizer that splits on single space characters and keeps all + * other characters, including sentence-final periods, attached to their token. Only + * the span-producing method is implemented because the adapter calls no other method. + */ + static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the adapter only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + private TestComponents() { + // Not instantiated; this class provides shared test fixtures only. + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index a72698e6f8..52b9e2288d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -24,6 +24,7 @@ import opennlp.tools.document.Annotation; import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; import opennlp.tools.document.LayerKey; import opennlp.tools.document.Layers; @@ -40,7 +41,7 @@ * * @since 3.0.0 */ -public class LemmatizerAnnotator implements DocumentAnnotator { +public final class LemmatizerAnnotator implements DocumentAnnotator { /** * The lemma layer. It is aligned with the token layer by position, and each annotation @@ -48,9 +49,6 @@ public class LemmatizerAnnotator implements DocumentAnnotator { */ public static final LayerKey LEMMAS = Layers.key("lemmas", String.class); - /** The message prefix of every absent-required-layer rejection in this adapter. */ - private static final String MISSING_LAYER = "document lacks the required layer "; - private final Lemmatizer lemmatizer; /** @@ -89,21 +87,8 @@ public LemmatizerAnnotator(Lemmatizer lemmatizer) { */ @Override public Document annotate(Document document) { - if (document == null) { - throw new IllegalArgumentException("document must not be null"); - } - if (!document.layers().contains(Layers.SENTENCES)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.SENTENCES); - } - if (!document.layers().contains(Layers.TOKENS)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.TOKENS); - } - if (!document.layers().contains(Layers.POS_TAGS)) { - throw new IllegalArgumentException(MISSING_LAYER - + Layers.POS_TAGS); - } + DocumentAnnotators.requireLayers(document, + Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); final List> sentences = document.get(Layers.SENTENCES); final List> tokens = document.get(Layers.TOKENS); final List> tags = document.get(Layers.POS_TAGS); @@ -112,47 +97,30 @@ public Document annotate(Document document) { + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } final List> layer = new ArrayList<>(tokens.size()); - // Walk the token layer once: both layers are in text order, so each sentence - // consumes the contiguous run of tokens whose spans it encloses. - int next = 0; - for (final Annotation sentence : sentences) { - final int first = next; - while (next < tokens.size() - && tokens.get(next).span().getStart() >= sentence.span().getStart() - && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { - next++; - } - final int count = next - first; - if (count == 0) { - continue; - } - final String[] words = new String[count]; - final String[] posTags = new String[count]; - for (int i = 0; i < count; i++) { - words[i] = tokens.get(first + i).value(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final String[] posTags = new String[words.length]; + for (int i = 0; i < words.length; i++) { posTags[i] = tags.get(first + i).value(); } final String[] lemmas = lemmatizer.lemmatize(words, posTags); - if (lemmas.length != count) { - throw new IllegalArgumentException( - "lemmatizer returned " + lemmas.length + " lemmas for " + count + " tokens"); + if (lemmas.length != words.length) { + throw new IllegalArgumentException("lemmatizer returned " + lemmas.length + + " lemmas for " + words.length + " tokens"); } - for (int i = 0; i < count; i++) { + for (int i = 0; i < words.length; i++) { layer.add(new Annotation<>(tokens.get(first + i).span(), lemmas[i])); } - } - if (next != tokens.size()) { - throw new IllegalArgumentException("token at " + tokens.get(next).span() - + " lies outside every sentence"); - } + }); return document.with(LEMMAS, layer); } + /** {@inheritDoc} */ @Override public Set> requires() { return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); } + /** {@inheritDoc} */ @Override public Set> provides() { return Set.of(LEMMAS); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index a8c00ba8f4..b5da3c4fe7 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -24,6 +24,7 @@ import opennlp.tools.document.Annotation; import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; import opennlp.tools.document.LayerKey; import opennlp.tools.document.Layers; @@ -36,7 +37,7 @@ * * @since 3.0.0 */ -public class StemmerAnnotator implements DocumentAnnotator { +public final class StemmerAnnotator implements DocumentAnnotator { /** * The stem layer. It is aligned with the token layer by position, and each annotation @@ -74,13 +75,7 @@ public StemmerAnnotator(Stemmer stemmer) { */ @Override public Document annotate(Document document) { - if (document == null) { - throw new IllegalArgumentException("document must not be null"); - } - if (!document.layers().contains(Layers.TOKENS)) { - throw new IllegalArgumentException("document lacks the required layer " - + Layers.TOKENS); - } + DocumentAnnotators.requireLayers(document, Layers.TOKENS); final List> tokens = document.get(Layers.TOKENS); final List> layer = new ArrayList<>(tokens.size()); for (final Annotation token : tokens) { @@ -89,11 +84,13 @@ public Document annotate(Document document) { return document.with(STEMS, layer); } + /** {@inheritDoc} */ @Override public Set> requires() { return Set.of(Layers.TOKENS); } + /** {@inheritDoc} */ @Override public Set> provides() { return Set.of(STEMS); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java index d31f092104..83a239751d 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Set; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import opennlp.tools.document.Annotation; @@ -29,6 +28,10 @@ import opennlp.tools.document.Layers; import opennlp.tools.util.Span; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class LemmatizerAnnotatorTest { /** Lowercases verbs and keeps everything else, enough to observe the adapter. */ @@ -65,10 +68,10 @@ void testLemmasAlignWithTokens() { final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); final List> lemmas = lemmatized.get(LemmatizerAnnotator.LEMMAS); - Assertions.assertEquals(3, lemmas.size()); - Assertions.assertEquals("run", lemmas.get(1).value()); - Assertions.assertEquals(new Span(4, 7), lemmas.get(1).span()); - Assertions.assertEquals("home", lemmas.get(2).value()); + assertEquals(3, lemmas.size()); + assertEquals("run", lemmas.get(1).value()); + assertEquals(new Span(4, 7), lemmas.get(1).span()); + assertEquals("home", lemmas.get(2).value()); } /** @@ -108,25 +111,25 @@ public List> lemmatize(List toks, List tags) { final Document lemmatized = new LemmatizerAnnotator(recording).annotate(document); - Assertions.assertEquals(List.of( + assertEquals(List.of( List.of("Ana", "runs."), List.of("Bob", "sits.")), calls); - Assertions.assertEquals(4, lemmatized.get(LemmatizerAnnotator.LEMMAS).size()); - Assertions.assertEquals(new Span(10, 13), + assertEquals(4, lemmatized.get(LemmatizerAnnotator.LEMMAS).size()); + assertEquals(new Span(10, 13), lemmatized.get(LemmatizerAnnotator.LEMMAS).get(2).span()); } @Test void testInvalidArguments() { - Assertions.assertThrows(IllegalArgumentException.class, + assertThrows(IllegalArgumentException.class, () -> new LemmatizerAnnotator(null)); final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); - Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); final Document misaligned = Document.of("a b") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 3), "a b"))) .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))) .with(Layers.POS_TAGS, List.of()); - Assertions.assertThrows(IllegalArgumentException.class, + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(misaligned)); } @@ -137,23 +140,23 @@ void testInvalidArguments() { @Test void testAbsentRequiredLayerThrowsWithExactMessage() { final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); - final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no layers"))); - Assertions.assertEquals("document lacks the required layer opennlp:sentences", + assertEquals("document lacks the required layer opennlp:sentences", e.getMessage()); final Document sentencesOnly = Document.of("a") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 1), "a"))); - final IllegalArgumentException tokenless = Assertions.assertThrows( + final IllegalArgumentException tokenless = assertThrows( IllegalArgumentException.class, () -> annotator.annotate(sentencesOnly)); - Assertions.assertEquals("document lacks the required layer opennlp:tokens", + assertEquals("document lacks the required layer opennlp:tokens", tokenless.getMessage()); final Document untagged = sentencesOnly .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))); - final IllegalArgumentException tagless = Assertions.assertThrows( + final IllegalArgumentException tagless = assertThrows( IllegalArgumentException.class, () -> annotator.annotate(untagged)); - Assertions.assertEquals("document lacks the required layer opennlp:pos", + assertEquals("document lacks the required layer opennlp:pos", tagless.getMessage()); } @@ -168,8 +171,8 @@ void testEmptyPresentLayersYieldEmptyLemmaLayer() { .with(Layers.TOKENS, List.of()) .with(Layers.POS_TAGS, List.of()); final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); - Assertions.assertTrue(lemmatized.layers().contains(LemmatizerAnnotator.LEMMAS)); - Assertions.assertTrue(lemmatized.get(LemmatizerAnnotator.LEMMAS).isEmpty()); + assertTrue(lemmatized.layers().contains(LemmatizerAnnotator.LEMMAS)); + assertTrue(lemmatized.get(LemmatizerAnnotator.LEMMAS).isEmpty()); } /** @@ -188,9 +191,9 @@ void testTokenOutsideEverySentenceThrowsWithExactMessage() { new Annotation<>(new Span(0, 3), "PROPN"), new Annotation<>(new Span(4, 9), "VERB"), new Annotation<>(new Span(10, 13), "PROPN"))); - final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new LemmatizerAnnotator(FIXTURE).annotate(document)); - Assertions.assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); + assertEquals("token at [10..13) lies outside every sentence", e.getMessage()); } /** @@ -218,9 +221,9 @@ public List> lemmatize(List toks, List tags) { .with(Layers.POS_TAGS, List.of( new Annotation<>(new Span(0, 1), "X"), new Annotation<>(new Span(2, 3), "X"))); - final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new LemmatizerAnnotator(shortLemmatizer).annotate(document)); - Assertions.assertEquals("lemmatizer returned 1 lemmas for 2 tokens", e.getMessage()); + assertEquals("lemmatizer returned 1 lemmas for 2 tokens", e.getMessage()); } /** @@ -229,7 +232,7 @@ public List> lemmatize(List toks, List tags) { */ @Test void testRequiresSentencesTokensAndTags() { - Assertions.assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), new LemmatizerAnnotator(FIXTURE).requires()); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java index 3f84b5303e..3d4493c247 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -19,7 +19,6 @@ import java.util.List; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import opennlp.tools.document.Annotation; @@ -27,6 +26,10 @@ import opennlp.tools.document.Layers; import opennlp.tools.util.Span; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class StemmerAnnotatorTest { @Test @@ -40,18 +43,18 @@ void testStemsAlignWithTokens() { new PorterStemmer()).annotate(document); final List> stems = stemmed.get(StemmerAnnotator.STEMS); - Assertions.assertEquals(2, stems.size()); - Assertions.assertEquals("run", stems.get(0).value()); - Assertions.assertEquals(new Span(0, 7), stems.get(0).span()); - Assertions.assertEquals("dog", stems.get(1).value()); + assertEquals(2, stems.size()); + assertEquals("run", stems.get(0).value()); + assertEquals(new Span(0, 7), stems.get(0).span()); + assertEquals("dog", stems.get(1).value()); } @Test void testInvalidArguments() { - Assertions.assertThrows(IllegalArgumentException.class, + assertThrows(IllegalArgumentException.class, () -> new StemmerAnnotator(null)); final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); - Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); } /** @@ -61,9 +64,9 @@ void testInvalidArguments() { @Test void testAbsentTokenLayerThrowsWithExactMessage() { final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); - final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no tokens"))); - Assertions.assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); + assertEquals("document lacks the required layer opennlp:tokens", e.getMessage()); } /** @@ -74,7 +77,7 @@ void testAbsentTokenLayerThrowsWithExactMessage() { void testEmptyPresentTokenLayerYieldsEmptyStemLayer() { final Document document = Document.of("").with(Layers.TOKENS, List.of()); final Document stemmed = new StemmerAnnotator(new PorterStemmer()).annotate(document); - Assertions.assertTrue(stemmed.layers().contains(StemmerAnnotator.STEMS)); - Assertions.assertTrue(stemmed.get(StemmerAnnotator.STEMS).isEmpty()); + assertTrue(stemmed.layers().contains(StemmerAnnotator.STEMS)); + assertTrue(stemmed.get(StemmerAnnotator.STEMS).isEmpty()); } } diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 29a009a884..648ee288a6 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -58,8 +58,9 @@ Immutable. Adding a layer returns a new - document that shares the unchanged layers with its ancestor. Documents are - safe to share between threads. Three invariants make index references + document that shares the unchanged layers with its ancestor. Documents + created through Document.of capture their text at construction + and are safe to share between threads. Three invariants make index references sound: a layer preserves its insertion order and is never sorted or reordered, a layer is immutable once added and detached from the caller's input list, and providing a layer that already exists is rejected, so an @@ -73,8 +74,7 @@ carries whole-document values without spans, which is where a language id, a category distribution, or provenance belongs: - - LANGUAGE = LayerKey.document("app:language", String.class); Document tagged = document.with(LANGUAGE, List.of(Annotation.of("eng"))); @@ -111,8 +111,7 @@ String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]> SentenceDetector, Tokenizer, and POSTagger, for example the ME implementations loaded from models: - - [18..23). Layers produced per token stay aligned with the token layer by position: - - > tokens = document.get(Layers.TOKENS); List> tags = document.get(Layers.POS_TAGS); for (int i = 0; i < tags.size(); i++) { @@ -160,8 +158,7 @@ for (Annotation token : tokens) { the token layer, and provides one integer annotation per token; the analyzer's build-time validation guarantees a tokenizer ran earlier: - - TOKEN_LENGTHS = LayerKey.of("token-lengths", Integer.class); @@ -169,9 +166,7 @@ class TokenLengthAnnotator implements DocumentAnnotator { @Override public Document annotate(Document document) { - if (document == null) { - throw new IllegalArgumentException("document must not be null"); - } + DocumentAnnotators.requireLayers(document, Layers.TOKENS); List> tokens = document.get(Layers.TOKENS); List> lengths = new ArrayList<>(tokens.size()); for (Annotation token : tokens) { @@ -196,8 +191,7 @@ class TokenLengthAnnotator implements DocumentAnnotator { numbers without a cast; for the text above the five values are 3, 3, 6, 2, 5, each on its token's span: - - > lengths = document.get(TOKEN_LENGTHS); int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> @@ -207,6 +201,9 @@ int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> gracefully on documents without content. An absent required layer is rejected with an IllegalArgumentException naming the layer, because a missing pipeline stage is an assembly error, not an empty document. + DocumentAnnotators.requireLayers performs exactly that rejection, + including the null check on the document, and is what the toolkit's own + adapters use.
From e9062250b1191c36e696a0a7490698e6f454b5bf Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:00:15 -0400 Subject: [PATCH 16/92] OPENNLP-1888: Address review: fold duplicate test fixtures and pin null rejection - Fold the three verbatim copies of the "Ana runs. Bob sits." document into a single twoSentenceDocument() helper in NameFinderAnnotatorTest, so the sentence and token layers of the shared fixture are declared once instead of drifting between the over-long mention, zero-length mention, and per-sentence offset tests - Hoist the no-op TokenNameFinder out of the blank-input test into a NO_NAMES constant in DocumentAnalyzerTest, since a finder that returns no spans is pipeline plumbing rather than part of any one test case, and document what it is for - Add testAnnotatorAdaptersRejectNullDocuments to pin that all four adapters reject a null document with the same "document must not be null" message, whether they validate directly or through DocumentAnnotators.requireLayers; the shared message was previously unpinned and free to drift per adapter - Trim the stale "person-free" qualifier from the New York comment in testTokenIndexSpansBecomeCharacterSpans; the finder emits a location mention and the extra negation described a distinction the test no longer draws --- .../tools/document/DocumentAnalyzerTest.java | 45 ++++++++++++----- .../document/NameFinderAnnotatorTest.java | 50 ++++++++----------- 2 files changed, 53 insertions(+), 42 deletions(-) diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java index 7b17c63692..1106f8f707 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -70,6 +70,19 @@ public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { } }; + /** A finder that finds no names, for tests that only exercise the pipeline plumbing. */ + private static final TokenNameFinder NO_NAMES = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + @Test void testPipelineProducesAlignedLayersInOriginalCoordinates() { final Document document = DocumentAnalyzer.builder() @@ -103,22 +116,11 @@ void testPipelineProducesAlignedLayersInOriginalCoordinates() { @ParameterizedTest @ValueSource(strings = {"", " "}) void testEmptyAndBlankInputProduceEmptyLayers(String text) { - final TokenNameFinder finder = new TokenNameFinder() { - - @Override - public Span[] find(String[] tokens) { - return new Span[0]; - } - - @Override - public void clearAdaptiveData() { - } - }; final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER)) .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)) .add(new POSTaggerAnnotator(TAGGER)) - .add(new NameFinderAnnotator(finder)) + .add(new NameFinderAnnotator(NO_NAMES)) .build(); final Document document = analyzer.analyze(text); @@ -194,4 +196,23 @@ void testAnnotatorAdaptersRejectNullDelegates() { assertThrows(IllegalArgumentException.class, () -> new POSTaggerAnnotator(null)); assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(null)); } + + /** + * Verifies that every adapter rejects a {@code null} document with the shared + * message, whether it checks itself or through + * {@link DocumentAnnotators#requireLayers(Document, LayerKey[])}. + */ + @Test + void testAnnotatorAdaptersRejectNullDocuments() { + final List adapters = List.of( + new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER), + new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER), + new POSTaggerAnnotator(TAGGER), + new NameFinderAnnotator(NO_NAMES)); + for (final DocumentAnnotator adapter : adapters) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> adapter.annotate(null)); + assertEquals("document must not be null", e.getMessage()); + } + } } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java index 3b3e87d5bb..e46be8bb2f 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -67,11 +67,27 @@ public void clearAdaptiveData() { }; } + /** + * @return A two-sentence document with sentence and token layers over + * {@code "Ana runs. Bob sits."}. Never {@code null}. + */ + private static Document twoSentenceDocument() { + return Document.of("Ana runs. Bob sits.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 9), "Ana runs."), + new Annotation<>(new Span(10, 19), "Bob sits."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "Ana"), + new Annotation<>(new Span(4, 9), "runs."), + new Annotation<>(new Span(10, 13), "Bob"), + new Annotation<>(new Span(14, 19), "sits."))); + } + @Test void testTokenIndexSpansBecomeCharacterSpans() { final AtomicInteger cleared = new AtomicInteger(); final TokenNameFinder finder = finder(tokens -> { - // "New York" as a two-token person-free location mention + // "New York" as a two-token location mention return new Span[] {new Span(1, 3, "location")}; }, cleared); @@ -146,15 +162,7 @@ void testMentionOutsideSentenceTokensFailsLoud() { // two tokens in the sentence, but the mention claims three return new Span[] {new Span(0, 3, "person")}; }, cleared); - final Document document = Document.of("Ana runs. Bob sits.") - .with(Layers.SENTENCES, List.of( - new Annotation<>(new Span(0, 9), "Ana runs."), - new Annotation<>(new Span(10, 19), "Bob sits."))) - .with(Layers.TOKENS, List.of( - new Annotation<>(new Span(0, 3), "Ana"), - new Annotation<>(new Span(4, 9), "runs."), - new Annotation<>(new Span(10, 13), "Bob"), - new Annotation<>(new Span(14, 19), "sits."))); + final Document document = twoSentenceDocument(); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(finder).annotate(document)); @@ -176,15 +184,7 @@ void testZeroLengthMentionFailsLoud() { final AtomicInteger cleared = new AtomicInteger(); final TokenNameFinder finder = finder(tokens -> "Bob".equals(tokens[0]) ? new Span[] {new Span(0, 0)} : new Span[0], cleared); - final Document document = Document.of("Ana runs. Bob sits.") - .with(Layers.SENTENCES, List.of( - new Annotation<>(new Span(0, 9), "Ana runs."), - new Annotation<>(new Span(10, 19), "Bob sits."))) - .with(Layers.TOKENS, List.of( - new Annotation<>(new Span(0, 3), "Ana"), - new Annotation<>(new Span(4, 9), "runs."), - new Annotation<>(new Span(10, 13), "Bob"), - new Annotation<>(new Span(14, 19), "sits."))); + final Document document = twoSentenceDocument(); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(finder).annotate(document)); @@ -231,17 +231,7 @@ void testFindsPerSentenceAndMapsSentenceLocalIndices() { return new Span[] {new Span(0, 1, "person")}; }, cleared); - final Document document = Document.of("Ana runs. Bob sits.") - .with(Layers.SENTENCES, List.of( - new Annotation<>(new Span(0, 9), "Ana runs."), - new Annotation<>(new Span(10, 19), "Bob sits."))) - .with(Layers.TOKENS, List.of( - new Annotation<>(new Span(0, 3), "Ana"), - new Annotation<>(new Span(4, 9), "runs."), - new Annotation<>(new Span(10, 13), "Bob"), - new Annotation<>(new Span(14, 19), "sits."))); - - final Document annotated = new NameFinderAnnotator(finder).annotate(document); + final Document annotated = new NameFinderAnnotator(finder).annotate(twoSentenceDocument()); assertEquals(List.of( List.of("Ana", "runs."), From 689636105f3a9b69a735442bb763e096bbae378e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:41:37 -0400 Subject: [PATCH 17/92] OPENNLP-1888: Name the null argument, avoid per-call key set wrapping, pin blank and span edge cases - ImmutableDocument: wrap the layer map unmodifiable at construction and expose its cached key set; split the combined null check so the message names the offending argument - StringUtil.isBlank javadoc: state how it differs from isUnicodeBlank - Tests: parameterize the isBlank accept and reject sides, pin the null NPE, and pin char-indexed spans over a supplementary-plane character --- .../tools/document/ImmutableDocument.java | 15 +++++--- .../java/opennlp/tools/util/StringUtil.java | 4 ++- .../tools/document/DocumentContractTest.java | 21 +++++++++++ .../opennlp/tools/util/StringUtilTest.java | 36 +++++++++++-------- 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index b456258d3d..dc23dd4106 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -43,7 +43,7 @@ final class ImmutableDocument implements Document { private ImmutableDocument(String text, Map, List>> layers) { this.text = text; - this.layers = layers; + this.layers = Collections.unmodifiableMap(layers); } /** @@ -86,14 +86,19 @@ public List> get(LayerKey layer) { /** {@inheritDoc} */ @Override public Set> layers() { - return Collections.unmodifiableSet(layers.keySet()); + // The unmodifiable map exposes an unmodifiable key set and caches it, so this + // accessor allocates no wrapper per call. + return layers.keySet(); } /** {@inheritDoc} */ @Override public Document with(LayerKey layer, List> annotations) { - if (layer == null || annotations == null) { - throw new IllegalArgumentException("layer and annotations must not be null"); + if (layer == null) { + throw new IllegalArgumentException("layer must not be null"); + } + if (annotations == null) { + throw new IllegalArgumentException("annotations must not be null"); } if (layers.containsKey(layer)) { throw new IllegalArgumentException("layer is already present: " + layer); @@ -123,6 +128,6 @@ public Document with(LayerKey layer, List> annotations) { } final Map, List>> grown = new LinkedHashMap<>(layers); grown.put(layer, List.copyOf(annotations)); - return new ImmutableDocument(text, Collections.unmodifiableMap(grown)); + return new ImmutableDocument(text, grown); } } diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 89694335cc..82041f7764 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -273,7 +273,9 @@ public static boolean isEmpty(CharSequence theString) { * code points that {@link #isWhitespace(int)} accepts. Unlike * {@link String#isBlank()}, this follows the toolkit's whitespace definition, which * includes the no-break spaces the JDK predicate leaves out, so a value spelled - * entirely from them cannot pass a blank check as content. + * entirely from them cannot pass a blank check as content. Unlike + * {@link #isUnicodeBlank(CharSequence)}, it resolves through the active + * {@link WhitespaceMode} and does not treat {@code null} as blank. * * @param theString The {@link CharSequence} to examine. Must not be {@code null}. * @return {@code true} if {@code theString} is empty or all whitespace. diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index fb01047aa5..75f62d34ea 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -222,6 +222,27 @@ void testZeroLengthSpansAreAccepted() { assertEquals(0, words.get(0).span().length()); } + /** + * Verifies that spans are indexed in {@code char} units, like {@link Span} itself: a + * supplementary-plane character counts as two, so a span over it covers the whole + * surrogate pair and the char-based text length is what bounds a span. + */ + @Test + void testSpansAreCharIndexedOverSupplementaryCharacters() { + // U+1F600, a supplementary-plane character, is two chars in the text + final String text = "\uD83D\uDE00 ok"; + final Document document = Document.of(text) + .with(WORDS, List.of( + new Annotation<>(new Span(0, 2), "emoji"), + new Annotation<>(new Span(3, 5), "ok"))); + final List> words = document.get(WORDS); + assertEquals("\uD83D\uDE00", + words.get(0).span().getCoveredText(document.text()).toString()); + assertEquals("ok", words.get(1).span().getCoveredText(document.text()).toString()); + assertThrows(IllegalArgumentException.class, () -> Document.of("\uD83D\uDE00") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "past the end")))); + } + /** * Verifies that a span reaching past the end of the text is rejected on insertion * with a message naming the span, the text length, and the layer. diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java index 73f81c209e..1951c114a0 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java @@ -681,21 +681,29 @@ void testLowercaseBeyondBMP() { } /** - * Verifies the blank check against the toolkit's whitespace definition: the - * no-break space is blank here although the JDK's own check does not cover it, - * whitespace-only and empty values are blank, and any non-whitespace code point, - * supplementary ones included, makes a value non-blank. + * Verifies the accepting side of the blank check against the toolkit's whitespace + * definition: empty and JDK-whitespace values are blank, and so are the no-break + * spaces U+00A0 and U+2007, which {@link String#isBlank()} does not cover. */ + @ParameterizedTest + @ValueSource(strings = {"", " \t\n", "\u00A0", " \u00A0\u2007 "}) + void testIsBlankAcceptsWhitespaceOnlyValues(String input) { + Assertions.assertTrue(StringUtil.isBlank(input)); + } + + /** + * Verifies the rejecting side of the blank check: any non-whitespace code point + * makes a value non-blank, including the supplementary-plane letter U+10428, which + * must be read as one code point rather than two chars. + */ + @ParameterizedTest + @ValueSource(strings = {"a", " a ", "\uD801\uDC28"}) + void testIsBlankRejectsValuesWithContent(String input) { + Assertions.assertFalse(StringUtil.isBlank(input)); + } + @Test - void testIsBlankFollowsTheToolkitWhitespaceDefinition() { - Assertions.assertTrue(StringUtil.isBlank("")); - Assertions.assertTrue(StringUtil.isBlank(" \t\n")); - // U+00A0 no-break space and U+2007 figure space: JDK String.isBlank says false - Assertions.assertTrue(StringUtil.isBlank("\u00A0")); - Assertions.assertTrue(StringUtil.isBlank(" \u00A0\u2007 ")); - Assertions.assertFalse(StringUtil.isBlank("a")); - Assertions.assertFalse(StringUtil.isBlank(" a ")); - // U+10428, a supplementary-plane letter read as one code point, not two chars - Assertions.assertFalse(StringUtil.isBlank("\uD801\uDC28")); + void testIsBlankWithNullString() { + Assertions.assertThrows(NullPointerException.class, () -> StringUtil.isBlank(null)); } } From 60843099016addb0d2590fe2fbf2aec1b3677493 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 16:46:58 -0400 Subject: [PATCH 18/92] OPENNLP-1888: Rework the document chapter after docs review Addresses rzo1's review comments on the manual: - Open with a plain-language definition and an inline typed-layer example instead of a bolded three-item list. - Show a stacked-layers figure for the running example up front and reference it from the pipeline section, so readers see the shape of a document before the API detail. - Explain span offsets, key identity, and the opennlp: prefix convention in prose a first-time reader can follow. - Reword the single-task API aside; drop the redundant custom annotator opener; cut the repeated statically-typed phrasing. --- opennlp-docs/src/docbkx/document.xml | 113 ++++++++++++++------------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 648ee288a6..53e69de2d2 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -21,53 +21,57 @@ Introduction The package opennlp.tools.document provides an immutable container - that carries the original text of one document together with any number of typed - annotation layers over it. Every annotation is anchored to a - Span of the text exactly as the caller supplied it, never to a - normalized or otherwise derived form, so any result of any pipeline step can be - highlighted in the source text. + that carries the original text of one document together with any number of + annotation layers over it. A layer is a list of Annotation values, + each pairing a Span with a value. A span's offsets count characters + from the beginning of the text exactly as the caller supplied it, never a + normalized or otherwise derived form, so every annotation can be highlighted in + the source text. The value's Java type is part of the layer's identity: a token + layer reads back as List<Annotation<String>>, for + example the token The paired with the span [0..3). + + + A document with three annotation layers + +text: The dog barks. It naps. + 0 1 2 + 01234567890123456789012 + +sentences: [0..14) "The dog barks." [15..23) "It naps." +tokens: [0..3) "The" [4..7) "dog" [8..14) "barks." [15..17) "It" [18..23) "naps." +pos-tags: [0..3) "DT" [4..7) "NN" [8..14) "VBZ" [15..17) "PRP" [18..23) "VBZ" + + + Each layer in the figure above is identified by a LayerKey that + names the layer and declares the type of its values: + Layers.TOKENS is a LayerKey<String>, so + document.get(Layers.TOKENS) returns the token annotations with + string values. Two keys are equal when their id, their value type, and their + scope are equal, so independently created constants for the same layer + interoperate. The keys of the core linguistic layers live in + Layers; a capability-specific layer's key lives on the annotator + that provides it, and adding a capability never changes the container. + + + Key ids are namespaced. Keys OpenNLP defines start with the + opennlp: prefix; a user extension needs to use its own prefix. A + bare id, like the token-lengths key below, is legal for an + application-local layer. Toolkit keys are created through + Layers.key and Layers.documentKey, which apply the + prefix. - The design follows three rules: + A document is never modified in place: adding a layer returns a new document + that shares the unchanged layers with its ancestor. Documents created through + Document.of capture their text at construction and are safe to + share between threads. Annotations reference other annotations by index within + their layer, never by object identity. Three invariants make those index + references sound: a layer preserves its insertion order and is never sorted or + reordered, a layer is + immutable once added and detached from the caller's input list, and providing + a layer that already exists is rejected, so an index reference stays valid for + the lifetime of the document. - - - - Offset-anchored. A layer is a list of - Annotation values, each pairing a Span in original - text coordinates with a typed value. Annotations reference other annotations - by index within their layer, never by object identity. - - - - - Open key space. A layer is identified by a - LayerKey that also carries the type of its values, so reading a - layer back is statically typed. Two keys are equal when their id, their - value type, and their scope are equal, so independently created constants - for the same layer interoperate. The keys of the core linguistic layers live in - Layers; a capability-specific layer's key lives on the annotator - that provides it, and adding a capability never changes the container. Ids - are namespaced: keys the toolkit defines carry the opennlp: - prefix, an extension uses its own prefix, and a bare id, like the - token-lengths key below, is legal for an application-local - layer. Toolkit keys are created through Layers.key and - Layers.documentKey, which apply the prefix. - - - - - Immutable. Adding a layer returns a new - document that shares the unchanged layers with its ancestor. Documents - created through Document.of capture their text at construction - and are safe to share between threads. Three invariants make index references - sound: a layer preserves its insertion order and is never sorted or - reordered, a layer is immutable once added and detached from the caller's - input list, and providing a layer that already exists is rejected, so an - index reference stays valid for the lifetime of the document. - - - A key declares its scope. Keys are positional by default: every annotation carries a span, and consumers never null-check it. A document-scoped key @@ -105,9 +109,10 @@ String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]> SentenceDetectorAnnotator, TokenizerAnnotator, POSTaggerAnnotator, NameFinderAnnotator, LemmatizerAnnotator, and StemmerAnnotator. Each wraps - the existing single-task API, which stays the primary API for single-task use. - The following pipeline combines three adapters with one custom annotator and - analyzes a two-sentence text; the components behind the adapters are any + one of the existing component APIs; an application that needs only one task + can keep using that component's API directly. The following pipeline combines + three adapters with one custom annotator and analyzes the text from the + introduction's figure; the components behind the adapters are any SentenceDetector, Tokenizer, and POSTagger, for example the ME implementations loaded from models: @@ -126,8 +131,9 @@ Document document = analyzer.analyze("The dog barks. It naps.");]]> round-trip shown here. - The resulting document carries exactly the four layers the pipeline provides. - The sentence layer holds two annotations, [0..14) covering + The resulting document carries exactly the four layers the pipeline provides, + rendered as the stacked layers of the introduction's figure. The sentence + layer holds two annotations, [0..14) covering The dog barks. and [15..23) covering It naps.. The token layer holds five tokens whose spans refer to the document text even inside the second sentence, so @@ -153,9 +159,8 @@ for (Annotation token : tokens) {
Writing a custom annotator - A new capability contributes its results as one more layer without any change to - the container. The annotator below defines its own key in its own code, reads - the token layer, and provides one integer annotation per token; the analyzer's + The annotator below defines its own key in its own code, reads the token + layer, and provides one integer annotation per token; the analyzer's build-time validation guarantees a tokenizer ran earlier: - Reading the layer back is statically typed by the key, so the values are used as - numbers without a cast; for the text above the five values are + Because the key declares the value type, the values come back as numbers + without a cast; for the text above the five values are 3, 3, 6, 2, 5, each on its token's span: Date: Mon, 17 Aug 2026 16:49:43 -0400 Subject: [PATCH 19/92] OPENNLP-1888: Pin the Document.merge contract Two contract tests fail red against the default method stub: java.lang.UnsupportedOperationException: merge is not implemented yet merge joins two documents grown independently over the same text, the parallel fan-out join rzo1 asked for on the pull request: disjoint layers stack into one document, the sources stay untouched, and a null argument, a different text, or a duplicate layer key is rejected with the offending key named. --- .../java/opennlp/tools/document/Document.java | 19 +++++++ .../tools/document/DocumentContractTest.java | 53 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 4169793eff..b03192eb8d 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -100,4 +100,23 @@ static Document of(CharSequence text) { * @throws IllegalArgumentException Thrown if any of the above constraints is violated. */ Document with(LayerKey layer, List> annotations); + + /** + * Returns a new document combining this document's layers with another document's + * layers over the same text: the parallel fan-out join. Two pipelines can each start + * from the same text and grow their own document independently; {@code merge} stacks + * their layers into one document. + * + * @param other The document whose layers are added on top of this document's layers. + * Must not be {@code null}, must carry the same text content, and must + * not provide a layer this document already has. + * @return A new {@link Document} carrying the layers of both documents. Never + * {@code null}; both source documents are left untouched. + * @throws IllegalArgumentException Thrown if {@code other} is {@code null}, if its + * text content differs, or if a layer key is present on both documents; the + * exception names the offending key. + */ + default Document merge(Document other) { + throw new UnsupportedOperationException("merge is not implemented yet"); + } } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index 75f62d34ea..d432a8d4b2 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -413,4 +413,57 @@ void testValueTypeTravelsThroughTheKey() { assertEquals("value of type java.lang.String does not match layer counts2", e.getMessage()); } + + /** + * Verifies that merge joins two documents grown independently over the same text, for + * example by two pipelines that ran in parallel, into one document carrying both layer + * sets, while leaving both sources untouched. Text content decides equality, not the + * {@link CharSequence} implementation. + */ + @Test + void testMergeJoinsLayersOfDocumentsOverTheSameText() { + final LayerKey lengths = LayerKey.of("lengths", Integer.class); + final Document words = Document.of("the dog") + .with(WORDS, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + final Document counted = Document.of(new StringBuilder("the dog")) + .with(lengths, List.of( + new Annotation<>(new Span(0, 3), 3), + new Annotation<>(new Span(4, 7), 3))); + + final Document merged = words.merge(counted); + + assertEquals("the dog", merged.text().toString()); + assertEquals(Set.of(WORDS, lengths), merged.layers()); + assertEquals("the", merged.get(WORDS).get(0).value()); + assertEquals(3, merged.get(lengths).get(0).value().intValue()); + // The sources are unchanged: merge, like with, never modifies in place. + assertEquals(Set.of(WORDS), words.layers()); + assertEquals(Set.of(lengths), counted.layers()); + } + + /** + * Verifies that merge rejects a null argument, a document over a different text even + * when their layers are disjoint, and a layer key present on both documents, naming + * the offending key. + */ + @Test + void testMergeRejectsNullDifferentTextAndDuplicateLayers() { + final Document words = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"))); + + final IllegalArgumentException nullOther = assertThrows(IllegalArgumentException.class, + () -> words.merge(null)); + assertEquals("other must not be null", nullOther.getMessage()); + + final IllegalArgumentException differentText = assertThrows( + IllegalArgumentException.class, () -> words.merge(Document.of("the cat"))); + assertEquals("merge requires both documents to carry the same text", + differentText.getMessage()); + + final IllegalArgumentException duplicate = assertThrows(IllegalArgumentException.class, + () -> words.merge(words)); + assertEquals("layer is already present: words", duplicate.getMessage()); + } } From 919cf590e9829ea5f46766000c7e1e499142f9c5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 16:50:30 -0400 Subject: [PATCH 20/92] OPENNLP-1888: Implement Document.merge as a default method The default body validates the argument and the shared text, then adds each of the other document's layers through with(), so every layer is re-validated against this document's contract and a duplicate key is rejected by the same once-only rule a direct add follows. The pinned contract tests now pass; opennlp-api is 386 tests, 0 failures. --- .../java/opennlp/tools/document/Document.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index b03192eb8d..962bb0b6de 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -117,6 +117,25 @@ static Document of(CharSequence text) { * exception names the offending key. */ default Document merge(Document other) { - throw new UnsupportedOperationException("merge is not implemented yet"); + if (other == null) { + throw new IllegalArgumentException("other must not be null"); + } + if (!text().toString().contentEquals(other.text())) { + throw new IllegalArgumentException( + "merge requires both documents to carry the same text"); + } + Document merged = this; + for (final LayerKey layer : other.layers()) { + merged = addLayer(merged, layer, other); + } + return merged; + } + + /** + * Adds one layer of {@code from} to {@code base}, capturing the key's value type so + * {@link #with(LayerKey, List)} re-validates the layer against this document. + */ + private static Document addLayer(Document base, LayerKey layer, Document from) { + return base.with(layer, from.get(layer)); } } From aa5aebae7d5b91a8467dc1d8a31e0b92f08088a9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 16:51:17 -0400 Subject: [PATCH 21/92] OPENNLP-1888: Document the parallel fan-out join in the manual --- opennlp-docs/src/docbkx/document.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 53e69de2d2..c2756c0847 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -154,6 +154,15 @@ for (Annotation token : tokens) { CharSequence covered = token.span().getCoveredText(document.text()); }]]> + + Because a document is immutable, independent pipelines can process the same + text in parallel, each starting from its own Document.of(text), + and their results can be joined afterwards: merge returns a new + document carrying the layers of both. The texts must match and the layer sets + must be disjoint, so two branches that both run a tokenizer collide on + opennlp:tokens; partition the branches by the layers they + provide. +
From adb15b42604aaf59f6bd690a353bf3e113a05e0c Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 18:27:48 -0400 Subject: [PATCH 22/92] OPENNLP-1888: Pin the duplicate-layer policy contract for merge Two contract tests fail red against the stubbed two-arg merge: java.lang.UnsupportedOperationException: merge with policy is not implemented yet The strict single-arg merge stays the default; the policy variant opts into keeping one copy of a layer both documents rebuilt identically, and still rejects differing copies with the key named. --- .../java/opennlp/tools/document/Document.java | 36 +++++++++++++++ .../tools/document/DocumentContractTest.java | 46 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 962bb0b6de..3d9e93a5b8 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -101,6 +101,23 @@ static Document of(CharSequence text) { */ Document with(LayerKey layer, List> annotations); + /** + * How {@link #merge(Document, DuplicateLayerPolicy)} treats a layer key that is + * present on both documents. + */ + enum DuplicateLayerPolicy { + + /** Reject any layer key present on both documents. */ + REJECT, + + /** + * Keep one copy of a layer key present on both documents when the two layers are + * structurally equal, for example when two parallel branches ran the same + * tokenizer. Layers whose contents differ are rejected as with {@link #REJECT}. + */ + KEEP_EQUAL + } + /** * Returns a new document combining this document's layers with another document's * layers over the same text: the parallel fan-out join. Two pipelines can each start @@ -131,6 +148,25 @@ default Document merge(Document other) { return merged; } + /** + * Returns a new document combining this document's layers with another document's + * layers over the same text, resolving duplicate layer keys with + * {@code duplicateLayers}. + * + * @param other The document whose layers are added on top of this document's layers. + * Must not be {@code null} and must carry the same text content. + * @param duplicateLayers How to treat a layer key present on both documents. Must + * not be {@code null}. + * @return A new {@link Document} carrying the layers of both documents. Never + * {@code null}; both source documents are left untouched. + * @throws IllegalArgumentException Thrown if either argument is {@code null}, if the + * text content differs, or if a layer key is present on both documents and + * the policy does not keep it; the exception names the offending key. + */ + default Document merge(Document other, DuplicateLayerPolicy duplicateLayers) { + throw new UnsupportedOperationException("merge with policy is not implemented yet"); + } + /** * Adds one layer of {@code from} to {@code base}, capturing the key's value type so * {@link #with(LayerKey, List)} re-validates the layer against this document. diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index d432a8d4b2..4a381176d2 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -466,4 +466,50 @@ void testMergeRejectsNullDifferentTextAndDuplicateLayers() { () -> words.merge(words)); assertEquals("layer is already present: words", duplicate.getMessage()); } + + /** + * Verifies that {@link Document.DuplicateLayerPolicy#KEEP_EQUAL} keeps one copy of a + * layer both documents rebuilt identically, for example the shared tokenizer prefix + * of two parallel branches, while still joining the disjoint layers. + */ + @Test + void testMergeKeepingEqualLayersToleratesIdenticalCopies() { + final List> tokens = List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog")); + final LayerKey lengths = LayerKey.of("lengths", Integer.class); + final Document words = Document.of("the dog").with(WORDS, tokens); + final Document recounted = Document.of("the dog") + .with(WORDS, tokens) + .with(lengths, List.of( + new Annotation<>(new Span(0, 3), 3), + new Annotation<>(new Span(4, 7), 3))); + + final Document merged = words.merge(recounted, Document.DuplicateLayerPolicy.KEEP_EQUAL); + + assertEquals(Set.of(WORDS, lengths), merged.layers()); + // The shared layer is kept once, not concatenated. + assertEquals(2, merged.get(WORDS).size()); + assertEquals(2, merged.get(lengths).size()); + } + + /** + * Verifies that {@link Document.DuplicateLayerPolicy#KEEP_EQUAL} still rejects a layer + * whose two copies differ, naming the key, and rejects a null policy. + */ + @Test + void testMergeKeepingEqualLayersRejectsDifferingCopiesAndNullPolicy() { + final Document words = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"))); + final Document retokenized = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 7), "the dog"))); + + final IllegalArgumentException differing = assertThrows(IllegalArgumentException.class, + () -> words.merge(retokenized, Document.DuplicateLayerPolicy.KEEP_EQUAL)); + assertEquals("layer is already present: words", differing.getMessage()); + + final IllegalArgumentException nullPolicy = assertThrows(IllegalArgumentException.class, + () -> words.merge(Document.of("the dog"), null)); + assertEquals("duplicateLayers must not be null", nullPolicy.getMessage()); + } } From 1297f725de40bc9dbd2c38ede78d6fe8eb90afc2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 18:29:03 -0400 Subject: [PATCH 23/92] OPENNLP-1888: Add a duplicate-layer policy to Document.merge merge(other) stays strict and now delegates to merge(other, REJECT). The KEEP_EQUAL policy keeps one copy of a layer both documents rebuilt identically, the shared sentence/tokenizer prefix of two parallel branches, while differing copies are still rejected with the key named. The pinned contract tests now pass; opennlp-api is 388 tests, 0 failures. The manual's fan-out paragraph documents the option. --- .../java/opennlp/tools/document/Document.java | 42 +++++++++++++------ opennlp-docs/src/docbkx/document.xml | 10 +++-- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 3d9e93a5b8..5f2165b498 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -134,18 +134,7 @@ enum DuplicateLayerPolicy { * exception names the offending key. */ default Document merge(Document other) { - if (other == null) { - throw new IllegalArgumentException("other must not be null"); - } - if (!text().toString().contentEquals(other.text())) { - throw new IllegalArgumentException( - "merge requires both documents to carry the same text"); - } - Document merged = this; - for (final LayerKey layer : other.layers()) { - merged = addLayer(merged, layer, other); - } - return merged; + return merge(other, DuplicateLayerPolicy.REJECT); } /** @@ -164,7 +153,34 @@ default Document merge(Document other) { * the policy does not keep it; the exception names the offending key. */ default Document merge(Document other, DuplicateLayerPolicy duplicateLayers) { - throw new UnsupportedOperationException("merge with policy is not implemented yet"); + if (other == null) { + throw new IllegalArgumentException("other must not be null"); + } + if (duplicateLayers == null) { + throw new IllegalArgumentException("duplicateLayers must not be null"); + } + if (!text().toString().contentEquals(other.text())) { + throw new IllegalArgumentException( + "merge requires both documents to carry the same text"); + } + Document merged = this; + for (final LayerKey layer : other.layers()) { + if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL + && merged.layers().contains(layer) + && layersEqual(merged, layer, other)) { + continue; + } + merged = addLayer(merged, layer, other); + } + return merged; + } + + /** + * @return Whether the layer is present on both documents with structurally equal + * contents. + */ + private static boolean layersEqual(Document first, LayerKey layer, Document second) { + return first.get(layer).equals(second.get(layer)); } /** diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index c2756c0847..52711508ed 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -158,10 +158,12 @@ for (Annotation token : tokens) { Because a document is immutable, independent pipelines can process the same text in parallel, each starting from its own Document.of(text), and their results can be joined afterwards: merge returns a new - document carrying the layers of both. The texts must match and the layer sets - must be disjoint, so two branches that both run a tokenizer collide on - opennlp:tokens; partition the branches by the layers they - provide. + document carrying the layers of both. The texts must match, and by default a + layer key present on both documents is rejected, so two branches that both + run a tokenizer collide on opennlp:tokens. When the branches + rebuild a shared prefix identically, passing + DuplicateLayerPolicy.KEEP_EQUAL keeps one copy of each agreeing + layer; copies whose contents differ are still rejected.
From 3245eff1c3b14f442d76a82bec56a208c954fb80 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 19:05:49 -0400 Subject: [PATCH 24/92] OPENNLP-1888: Render the layer figure monospaced and tighten the prose - literallayout class=monospaced makes the docbkx toolchain emit a pre block, so the figure's character ruler and layer rows column-align; plain literallayout renders in the proportional body font. - Correct the pipeline section: the figure shows three of the four layers; the custom token-lengths layer is the fourth. - Trim restated clauses in the merge javadoc, the contract test javadoc, and the introduction; align the layersEqual and addLayer helper javadoc with what the helpers do. --- .../java/opennlp/tools/document/Document.java | 12 +++++------- .../tools/document/DocumentContractTest.java | 9 +++------ opennlp-docs/src/docbkx/document.xml | 19 +++++++++---------- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 5f2165b498..c78d5d3b8f 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -120,9 +120,8 @@ enum DuplicateLayerPolicy { /** * Returns a new document combining this document's layers with another document's - * layers over the same text: the parallel fan-out join. Two pipelines can each start - * from the same text and grow their own document independently; {@code merge} stacks - * their layers into one document. + * layers over the same text, joining documents grown independently, for example by + * pipelines that ran in parallel. * * @param other The document whose layers are added on top of this document's layers. * Must not be {@code null}, must carry the same text content, and must @@ -176,16 +175,15 @@ && layersEqual(merged, layer, other)) { } /** - * @return Whether the layer is present on both documents with structurally equal - * contents. + * @return Whether the two documents carry structurally equal contents for the layer. */ private static boolean layersEqual(Document first, LayerKey layer, Document second) { return first.get(layer).equals(second.get(layer)); } /** - * Adds one layer of {@code from} to {@code base}, capturing the key's value type so - * {@link #with(LayerKey, List)} re-validates the layer against this document. + * Adds one layer of {@code from} to {@code base} through {@link #with(LayerKey, List)}, + * capturing the key's value type. */ private static Document addLayer(Document base, LayerKey layer, Document from) { return base.with(layer, from.get(layer)); diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index 4a381176d2..3fe70e175f 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -415,9 +415,8 @@ void testValueTypeTravelsThroughTheKey() { } /** - * Verifies that merge joins two documents grown independently over the same text, for - * example by two pipelines that ran in parallel, into one document carrying both layer - * sets, while leaving both sources untouched. Text content decides equality, not the + * Verifies that merge joins two documents grown independently over the same text and + * leaves both sources untouched. Text content decides equality, not the * {@link CharSequence} implementation. */ @Test @@ -438,7 +437,6 @@ void testMergeJoinsLayersOfDocumentsOverTheSameText() { assertEquals(Set.of(WORDS, lengths), merged.layers()); assertEquals("the", merged.get(WORDS).get(0).value()); assertEquals(3, merged.get(lengths).get(0).value().intValue()); - // The sources are unchanged: merge, like with, never modifies in place. assertEquals(Set.of(WORDS), words.layers()); assertEquals(Set.of(lengths), counted.layers()); } @@ -469,8 +467,7 @@ void testMergeRejectsNullDifferentTextAndDuplicateLayers() { /** * Verifies that {@link Document.DuplicateLayerPolicy#KEEP_EQUAL} keeps one copy of a - * layer both documents rebuilt identically, for example the shared tokenizer prefix - * of two parallel branches, while still joining the disjoint layers. + * layer both documents rebuilt identically while still joining the disjoint layers. */ @Test void testMergeKeepingEqualLayersToleratesIdenticalCopies() { diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 52711508ed..9e21bb73a7 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -25,14 +25,14 @@ annotation layers over it. A layer is a list of Annotation values, each pairing a Span with a value. A span's offsets count characters from the beginning of the text exactly as the caller supplied it, never a - normalized or otherwise derived form, so every annotation can be highlighted in + normalized or derived form, so every annotation can be highlighted in the source text. The value's Java type is part of the layer's identity: a token layer reads back as List<Annotation<String>>, for example the token The paired with the span [0..3). A document with three annotation layers - + text: The dog barks. It naps. 0 1 2 01234567890123456789012 @@ -54,7 +54,7 @@ pos-tags: [0..3) "DT" [4..7) "NN" [8..14) "VBZ" [15..17) "PRP" [18..23) "V Key ids are namespaced. Keys OpenNLP defines start with the - opennlp: prefix; a user extension needs to use its own prefix. A + opennlp: prefix; a user extension must use its own prefix. A bare id, like the token-lengths key below, is legal for an application-local layer. Toolkit keys are created through Layers.key and Layers.documentKey, which apply the @@ -66,11 +66,10 @@ pos-tags: [0..3) "DT" [4..7) "NN" [8..14) "VBZ" [15..17) "PRP" [18..23) "V Document.of capture their text at construction and are safe to share between threads. Annotations reference other annotations by index within their layer, never by object identity. Three invariants make those index - references sound: a layer preserves its insertion order and is never sorted or - reordered, a layer is - immutable once added and detached from the caller's input list, and providing - a layer that already exists is rejected, so an index reference stays valid for - the lifetime of the document. + references sound: a layer keeps its insertion order, a layer is immutable once + added and detached from the caller's input list, and providing a layer that + already exists is rejected, so an index reference stays valid for the lifetime + of the document. A key declares its scope. Keys are positional by default: every annotation @@ -131,8 +130,8 @@ Document document = analyzer.analyze("The dog barks. It naps.");]]> round-trip shown here. - The resulting document carries exactly the four layers the pipeline provides, - rendered as the stacked layers of the introduction's figure. The sentence + The resulting document carries exactly the four layers the pipeline provides: + the figure's three plus the custom token-lengths layer. The sentence layer holds two annotations, [0..14) covering The dog barks. and [15..23) covering It naps.. The token layer holds five tokens whose spans refer to From 4f5c0cacda1dd8050ffe3e84045584179608f8b2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 19:20:40 -0400 Subject: [PATCH 25/92] OPENNLP-1888: Pin a differing-contents message for KEEP_EQUAL rejection The repinned test fails red: a KEEP_EQUAL merge that rejects a layer whose copies differ still reports 'layer is already present', which reads as if the policy was ignored. The caller opted into duplicates; the reason worth naming is that the contents differ. --- .../test/java/opennlp/tools/document/DocumentContractTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index 3fe70e175f..cd4fc7d132 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -503,7 +503,8 @@ void testMergeKeepingEqualLayersRejectsDifferingCopiesAndNullPolicy() { final IllegalArgumentException differing = assertThrows(IllegalArgumentException.class, () -> words.merge(retokenized, Document.DuplicateLayerPolicy.KEEP_EQUAL)); - assertEquals("layer is already present: words", differing.getMessage()); + assertEquals("layer is present on both documents with differing contents: words", + differing.getMessage()); final IllegalArgumentException nullPolicy = assertThrows(IllegalArgumentException.class, () -> words.merge(Document.of("the dog"), null)); From 844128fad9d65796a9ef155db45676c2aac12861 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 19:21:12 -0400 Subject: [PATCH 26/92] OPENNLP-1888: Name the differing contents when KEEP_EQUAL rejects a layer When the policy is KEEP_EQUAL and a layer key is present on both documents, a failed equality check now throws directly instead of falling through to with(), so the message states the actual reason: the copies differ, not merely that the key is a duplicate. The pinned contract test passes; opennlp-api is 388 tests, 0 failures. --- .../src/main/java/opennlp/tools/document/Document.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index c78d5d3b8f..1e3d8d40a3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -165,9 +165,12 @@ default Document merge(Document other, DuplicateLayerPolicy duplicateLayers) { Document merged = this; for (final LayerKey layer : other.layers()) { if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL - && merged.layers().contains(layer) - && layersEqual(merged, layer, other)) { - continue; + && merged.layers().contains(layer)) { + if (layersEqual(merged, layer, other)) { + continue; + } + throw new IllegalArgumentException( + "layer is present on both documents with differing contents: " + layer); } merged = addLayer(merged, layer, other); } From 75fcac098ab5a0bc34af54c35f13657bc07cc8ca Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 19:21:33 -0400 Subject: [PATCH 27/92] OPENNLP-1888: Pin what layer equality compares for KEEP_EQUAL One javadoc sentence on the constant: equality is Annotation equality, so spans compare by offsets and type, never by probability, and values by their own equals. Two branches running different models over the same text can therefore agree; the kept copy is this document's. --- opennlp-api/src/main/java/opennlp/tools/document/Document.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java index 1e3d8d40a3..4d22d3a18b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Document.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -114,6 +114,8 @@ enum DuplicateLayerPolicy { * Keep one copy of a layer key present on both documents when the two layers are * structurally equal, for example when two parallel branches ran the same * tokenizer. Layers whose contents differ are rejected as with {@link #REJECT}. + * Equality is {@link Annotation} equality: spans compare by offsets and type, + * never by probability, and values by their own {@code equals}. */ KEEP_EQUAL } From 562af8714b4f800bd97a5a8c88a267bd5dafd6a5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 19:33:27 -0400 Subject: [PATCH 28/92] OPENNLP-1888: Pin merge semantics across Document implementations Two guards ahead of an ImmutableDocument merge override: the interface default serves implementations that do not override merge with the same join, KEEP_EQUAL, and rejection messages, and merge re-validates the layers it takes from a foreign document instead of trusting them, rejecting an out-of-bounds span by name. --- .../tools/document/DocumentContractTest.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index cd4fc7d132..e83a01508c 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -510,4 +510,107 @@ void testMergeKeepingEqualLayersRejectsDifferingCopiesAndNullPolicy() { () -> words.merge(Document.of("the dog"), null)); assertEquals("duplicateLayers must not be null", nullPolicy.getMessage()); } + + /** + * Verifies that a document implementation that does not override merge gets the same + * semantics from the interface default: disjoint layers join, a layer both documents + * rebuilt identically is kept once under KEEP_EQUAL, and differing copies are rejected + * with the same message the default implementation produces. + */ + @Test + void testMergeDefaultImplementationServesForeignDocuments() { + final List> tokens = List.of(new Annotation<>(new Span(0, 3), "the")); + final LayerKey lengths = LayerKey.of("lengths", Integer.class); + final Document words = new DelegatingDocument(Document.of("the dog").with(WORDS, tokens)); + final Document counted = Document.of("the dog") + .with(WORDS, tokens) + .with(lengths, List.of(new Annotation<>(new Span(0, 3), 3))); + + final Document merged = words.merge(counted, Document.DuplicateLayerPolicy.KEEP_EQUAL); + assertEquals(Set.of(WORDS, lengths), merged.layers()); + assertEquals(1, merged.get(WORDS).size()); + + final Document retokenized = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 7), "the dog"))); + final IllegalArgumentException differing = assertThrows(IllegalArgumentException.class, + () -> words.merge(retokenized, Document.DuplicateLayerPolicy.KEEP_EQUAL)); + assertEquals("layer is present on both documents with differing contents: words", + differing.getMessage()); + } + + /** + * Verifies that merge validates the layers it takes from the other document instead of + * trusting them: a foreign implementation can hand out annotations that were never + * checked, and an out-of-bounds span among them is rejected by name. + */ + @Test + void testMergeRevalidatesLayersOfForeignDocuments() { + final Document words = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"))); + final LayerKey stale = LayerKey.of("stale", String.class); + final Document unvalidated = new UnvalidatedDocument("the dog", stale, + List.of(new Annotation<>(new Span(0, 99), "out of bounds"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> words.merge(unvalidated)); + assertEquals("span [0..99) exceeds the text length 7 in layer stale", + e.getMessage()); + } + + /** + * A pass-through wrapper that overrides none of the interface defaults, so calling + * merge on it runs the interface's default implementation. + */ + private record DelegatingDocument(Document delegate) implements Document { + + @Override + public CharSequence text() { + return delegate.text(); + } + + @Override + public List> get(LayerKey layer) { + return delegate.get(layer); + } + + @Override + public Set> layers() { + return delegate.layers(); + } + + @Override + public Document with(LayerKey layer, List> annotations) { + return new DelegatingDocument(delegate.with(layer, annotations)); + } + } + + /** + * A document whose single layer bypassed all validation, standing in for a foreign + * implementation that does not enforce the layer contract itself. + */ + private record UnvalidatedDocument(String rawText, LayerKey key, + List> annotations) + implements Document { + + @Override + public CharSequence text() { + return rawText; + } + + @Override + @SuppressWarnings("unchecked") + public List> get(LayerKey layer) { + return key.equals(layer) ? (List>) (List) annotations : List.of(); + } + + @Override + public Set> layers() { + return Set.of(key); + } + + @Override + public Document with(LayerKey layer, List> annotations) { + throw new UnsupportedOperationException(); + } + } } From 09cd47d59ed2e8291b03de6b452195e88cc48f07 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 19:34:34 -0400 Subject: [PATCH 29/92] OPENNLP-1888: Merge with a single map copy in ImmutableDocument The interface default adds the other document's layers through with(), building one intermediate document and one map copy per layer. The override validates each incoming layer with the same checks with() runs, then copies the layer map once and allocates one document; when nothing was added it returns this, matching the default. The layer validation moves from with() into a shared helper unchanged. Pinned by the cross-implementation contract tests; opennlp-api is 390 tests, 0 failures, runtime annotator suites green. --- .../tools/document/ImmutableDocument.java | 66 ++++++++++++++++++- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index dc23dd4106..e175c05f6c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -103,6 +103,69 @@ public Document with(LayerKey layer, List> annotations) { if (layers.containsKey(layer)) { throw new IllegalArgumentException("layer is already present: " + layer); } + validate(layer, annotations); + final Map, List>> grown = new LinkedHashMap<>(layers); + grown.put(layer, List.copyOf(annotations)); + return new ImmutableDocument(text, grown); + } + + /** + * {@inheritDoc} + * This implementation copies the layer map once, not once per added layer. + */ + @Override + public Document merge(Document other, DuplicateLayerPolicy duplicateLayers) { + if (other == null) { + throw new IllegalArgumentException("other must not be null"); + } + if (duplicateLayers == null) { + throw new IllegalArgumentException("duplicateLayers must not be null"); + } + if (!text.contentEquals(other.text())) { + throw new IllegalArgumentException( + "merge requires both documents to carry the same text"); + } + final Map, List>> combined = new LinkedHashMap<>(layers); + for (final LayerKey layer : other.layers()) { + if (combined.containsKey(layer)) { + if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL + && get(layer).equals(other.get(layer))) { + continue; + } + throw new IllegalArgumentException(duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL + ? "layer is present on both documents with differing contents: " + layer + : "layer is already present: " + layer); + } + combined.put(layer, copyValidated(layer, other)); + } + if (combined.size() == layers.size()) { + return this; + } + return new ImmutableDocument(text, combined); + } + + /** + * {@return a validated immutable copy of one of {@code from}'s layers, capturing the + * key's value type} + */ + private List> copyValidated(LayerKey layer, Document from) { + final List> annotations = from.get(layer); + if (annotations == null) { + throw new IllegalArgumentException("annotations must not be null"); + } + validate(layer, annotations); + return List.copyOf(annotations); + } + + /** + * Checks one layer's annotations against the key's contract: no null elements, values + * assignable to the key's type, spans present and within the text bounds under a + * positional key, absent under a document-scoped key. + * + * @throws IllegalArgumentException Thrown if any check fails; the message names the + * layer. + */ + private void validate(LayerKey layer, List> annotations) { for (final Annotation annotation : annotations) { if (annotation == null) { throw new IllegalArgumentException("annotations must not contain null: " + layer); @@ -126,8 +189,5 @@ public Document with(LayerKey layer, List> annotations) { "document-scoped layer " + layer + " must not carry spans"); } } - final Map, List>> grown = new LinkedHashMap<>(layers); - grown.put(layer, List.copyOf(annotations)); - return new ImmutableDocument(text, grown); } } From 0705c5d7373df1b7be051139fa5fca4b2d7c9156 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 20:12:16 -0400 Subject: [PATCH 30/92] OPENNLP-1888: State span offsets in UTF-16 units in the manual The chapter said offsets count characters; the pinned contract test shows a supplementary-plane character counts as two. Say Java chars (UTF-16 units) so the claim matches the tested behavior. --- opennlp-docs/src/docbkx/document.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 9e21bb73a7..9a1aa92410 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -23,8 +23,9 @@ The package opennlp.tools.document provides an immutable container that carries the original text of one document together with any number of annotation layers over it. A layer is a list of Annotation values, - each pairing a Span with a value. A span's offsets count characters - from the beginning of the text exactly as the caller supplied it, never a + each pairing a Span with a value. A span's offsets count Java + chars (UTF-16 units, so a supplementary-plane character counts as two) from + the beginning of the text exactly as the caller supplied it, never a normalized or derived form, so every annotation can be highlighted in the source text. The value's Java type is part of the layer's identity: a token layer reads back as List<Annotation<String>>, for From 550bc449063c0ba00c6343feb7975fdc56cce902 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 20:19:02 -0400 Subject: [PATCH 31/92] OPENNLP-1888: Pin that adapters name themselves by class name Three tests fail red: the adapters inherit the identity toString, so a pipeline validation message reads 'annotator opennlp.tools.document.SentenceDetectorAnnotator@3b96c42e requires layer ...' instead of naming the adapter. The stemmer test pins the full analyzer message exactly, since that adapter requires a single layer and the message is therefore deterministic. --- .../tools/document/DocumentAnalyzerTest.java | 14 ++++++++++++++ .../tools/lemmatizer/LemmatizerAnnotatorTest.java | 9 +++++++++ .../tools/stemmer/StemmerAnnotatorTest.java | 14 ++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java index 1106f8f707..b6d9234b66 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -83,6 +83,20 @@ public void clearAdaptiveData() { } }; + /** + * Verifies that the adapters identify themselves by their simple class name, which is + * how a pipeline validation message names the offending annotator. + */ + @Test + void testAdaptersNameThemselvesByClassName() { + assertEquals("SentenceDetectorAnnotator", + new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER).toString()); + assertEquals("TokenizerAnnotator", + new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER).toString()); + assertEquals("POSTaggerAnnotator", new POSTaggerAnnotator(TAGGER).toString()); + assertEquals("NameFinderAnnotator", new NameFinderAnnotator(NO_NAMES).toString()); + } + @Test void testPipelineProducesAlignedLayersInOriginalCoordinates() { final Document document = DocumentAnalyzer.builder() diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java index 83a239751d..63654bbada 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -119,6 +119,15 @@ public List> lemmatize(List toks, List tags) { lemmatized.get(LemmatizerAnnotator.LEMMAS).get(2).span()); } + /** + * Verifies that the adapter identifies itself by its simple class name, which is how a + * pipeline validation message names the offending annotator. + */ + @Test + void testAdapterNamesItselfByClassName() { + assertEquals("LemmatizerAnnotator", new LemmatizerAnnotator(FIXTURE).toString()); + } + @Test void testInvalidArguments() { assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java index 3d4493c247..668acd901c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -23,6 +23,7 @@ import opennlp.tools.document.Annotation; import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; import opennlp.tools.document.Layers; import opennlp.tools.util.Span; @@ -57,6 +58,19 @@ void testInvalidArguments() { assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); } + /** + * Verifies that a misordered pipeline names the adapter by its simple class name, not + * by its default identity string. + */ + @Test + void testPipelineValidationNamesTheAdapter() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnalyzer.builder() + .add(new StemmerAnnotator(new PorterStemmer())).build()); + assertEquals("annotator StemmerAnnotator requires layer opennlp:tokens," + + " which no earlier annotator provides", e.getMessage()); + } + /** * Verifies that a document without a token layer is rejected with a message naming the * missing layer, instead of silently producing an empty stem layer. From f4dd343a9ca08559ef382923799a31e9477cd311 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 17 Aug 2026 20:20:46 -0400 Subject: [PATCH 32/92] OPENNLP-1888: Name adapters by their simple class name All six adapters override toString with the simple class name, so a pipeline validation message names the offending annotator readably. The pinned tests pass; opennlp-api and opennlp-runtime suites green. --- .../java/opennlp/tools/document/NameFinderAnnotator.java | 9 +++++++++ .../java/opennlp/tools/document/POSTaggerAnnotator.java | 9 +++++++++ .../tools/document/SentenceDetectorAnnotator.java | 9 +++++++++ .../java/opennlp/tools/document/TokenizerAnnotator.java | 9 +++++++++ .../opennlp/tools/lemmatizer/LemmatizerAnnotator.java | 9 +++++++++ .../java/opennlp/tools/stemmer/StemmerAnnotator.java | 9 +++++++++ 6 files changed, 54 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java index 677e0dfa20..97e9d9526d 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -133,4 +133,13 @@ public Set> requires() { public Set> provides() { return Set.of(Layers.ENTITIES); } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java index b97aa98b4b..82facfba2b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -101,4 +101,13 @@ public Set> requires() { public Set> provides() { return Set.of(Layers.POS_TAGS); } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java index a035f0517e..183a6df13e 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java @@ -78,4 +78,13 @@ public Document annotate(Document document) { public Set> provides() { return Set.of(Layers.SENTENCES); } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java index 6a81fcea9c..b7d6b9e288 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -100,4 +100,13 @@ private void addTokens(List> tokens, String text, int offset) public Set> provides() { return Set.of(Layers.TOKENS); } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index 52b9e2288d..2c633798e5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -125,4 +125,13 @@ public Set> requires() { public Set> provides() { return Set.of(LEMMAS); } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index b5da3c4fe7..900fe188a6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -95,4 +95,13 @@ public Set> requires() { public Set> provides() { return Set.of(STEMS); } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } } From ffb99d009cab86f386b430c5caa60f4f98199d53 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 14 Jul 2026 21:09:58 -0400 Subject: [PATCH 33/92] depparse: Arc-standard dependency parser skeleton with CoNLL-U reader and UAS/LAS evaluator Adds opennlp.tools.depparse: the DependencyParser interface with DependencyArc, DependencyGraph, and DependencySample in opennlp-api, and a greedy transition-based implementation in the runtime: arc-standard state and static oracle, configuration features, an oracle-driven event stream, DependencyParserME over the existing maxent machinery, and a UAS/LAS evaluator. ConlluDependencySampleStream maps the basic dependency columns of CoNLL-U sentences so Universal Dependencies treebanks train directly; sentences whose multiword tokens were merged by ConlluStream are skipped because their syntactic words are not recoverable. Non-projective trees have no arc-standard derivation and are skipped during event generation. Model persistence and a DL-backed implementation are follow-ups behind the same interface. (cherry picked from commit 7448e34080480cc52b266069f5c700557957085c) --- .../opennlp/tools/depparse/DependencyArc.java | 63 +++++ .../tools/depparse/DependencyGraph.java | 179 ++++++++++++++ .../tools/depparse/DependencyParser.java | 47 ++++ .../tools/depparse/DependencySample.java | 113 +++++++++ .../tools/depparse/DependencyGraphTest.java | 140 +++++++++++ .../tools/depparse/DependencySampleTest.java | 78 +++++++ .../conllu/ConlluDependencySampleStream.java | 131 +++++++++++ .../ConlluDependencySampleStreamTest.java | 105 +++++++++ .../tools/depparse/ArcStandardOracle.java | 104 +++++++++ .../tools/depparse/ArcStandardState.java | 219 ++++++++++++++++++ .../depparse/DependencyContextGenerator.java | 115 +++++++++ .../tools/depparse/DependencyEvaluator.java | 88 +++++++ .../tools/depparse/DependencyEventStream.java | 107 +++++++++ .../tools/depparse/DependencyParserME.java | 140 +++++++++++ .../opennlp/tools/depparse/Transition.java | 127 ++++++++++ .../tools/depparse/ArcStandardOracleTest.java | 87 +++++++ .../depparse/DependencyParserMETest.java | 122 ++++++++++ 17 files changed, 1965 insertions(+) create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java create mode 100644 opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java new file mode 100644 index 0000000000..bf9dc2eea4 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * One labeled edge of a {@link DependencyGraph}: the token at {@code dependent} is governed + * by the token at {@code head} under the given {@code relation}. + * + *

Indices are zero-based positions in the token array the graph was built over. A + * {@code head} of {@link #ROOT_HEAD} marks the dependent as the sentence root, which is + * attached to the artificial root node rather than to another token.

+ * + * @param head The zero-based index of the governing token, or {@link #ROOT_HEAD} when the + * dependent is the sentence root. + * @param dependent The zero-based index of the governed token. + * @param relation The dependency relation label, for example {@code nsubj}. + * + * @since 3.0.0 + */ +public record DependencyArc(int head, int dependent, String relation) { + + /** + * The {@code head} value marking an arc from the artificial root node. + */ + public static final int ROOT_HEAD = -1; + + /** + * Validates the arc invariants. + * + * @throws IllegalArgumentException Thrown if {@code dependent} is negative, {@code head} + * is less than {@link #ROOT_HEAD}, the arc is a self-loop, or {@code relation} + * is {@code null} or blank. + */ + public DependencyArc { + if (dependent < 0) { + throw new IllegalArgumentException("dependent must not be negative: " + dependent); + } + if (head < ROOT_HEAD) { + throw new IllegalArgumentException("head must be a token index or ROOT_HEAD: " + head); + } + if (head == dependent) { + throw new IllegalArgumentException("arc must not be a self-loop: " + head); + } + if (relation == null || relation.isBlank()) { + throw new IllegalArgumentException("relation must not be null or blank"); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java new file mode 100644 index 0000000000..e95234dcc8 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * An immutable dependency tree over one sentence: for every token, the index of its head + * and the label of the relation to that head. + * + *

Token indices are zero-based positions in the sentence the graph was built for. + * Exactly one token carries the head value {@link DependencyArc#ROOT_HEAD}, marking it as + * the sentence root. Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public final class DependencyGraph { + + private final int[] heads; + private final String[] relations; + + private DependencyGraph(int[] heads, String[] relations) { + this.heads = heads; + this.relations = relations; + } + + /** + * Creates a {@link DependencyGraph} from parallel head and relation arrays. + * + * @param heads For each token, the zero-based index of its head token, or + * {@link DependencyArc#ROOT_HEAD} for the sentence root. Must not be + * {@code null} or empty, every value must be a valid token index or + * {@link DependencyArc#ROOT_HEAD}, no token may head itself, and exactly + * one token must be the root. + * @param relations For each token, the label of the relation to its head. Must not be + * {@code null}, must have the same length as {@code heads}, and no + * entry may be {@code null} or blank. + * @return A validated {@link DependencyGraph}. Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + public static DependencyGraph of(int[] heads, String[] relations) { + if (heads == null || relations == null) { + throw new IllegalArgumentException("heads and relations must not be null"); + } + if (heads.length == 0) { + throw new IllegalArgumentException("a dependency graph needs at least one token"); + } + if (heads.length != relations.length) { + throw new IllegalArgumentException("heads and relations must have the same length: " + + heads.length + " != " + relations.length); + } + int roots = 0; + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + roots++; + } else if (heads[i] < 0 || heads[i] >= heads.length) { + throw new IllegalArgumentException("head of token " + i + + " is out of range: " + heads[i]); + } else if (heads[i] == i) { + throw new IllegalArgumentException("token " + i + " must not head itself"); + } + if (relations[i] == null || relations[i].isBlank()) { + throw new IllegalArgumentException("relation of token " + i + " must not be blank"); + } + } + if (roots != 1) { + throw new IllegalArgumentException("expected exactly one root, found " + roots); + } + return new DependencyGraph(heads.clone(), relations.clone()); + } + + /** + * @return The number of tokens the graph spans. + */ + public int size() { + return heads.length; + } + + /** + * Retrieves the head of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The zero-based index of the head token, or {@link DependencyArc#ROOT_HEAD} + * when the token is the sentence root. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int headOf(int index) { + checkIndex(index); + return heads[index]; + } + + /** + * Retrieves the relation label of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The label of the relation between the token and its head. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String relationOf(int index) { + checkIndex(index); + return relations[index]; + } + + /** + * @return The zero-based index of the sentence root token. + */ + public int root() { + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + return i; + } + } + throw new IllegalStateException("graph invariant violated: no root present"); + } + + /** + * @return All arcs of the graph in token order, one per token. Never {@code null}. + */ + public List arcs() { + final List arcs = new ArrayList<>(heads.length); + for (int i = 0; i < heads.length; i++) { + arcs.add(new DependencyArc(heads[i], i, relations[i])); + } + return Collections.unmodifiableList(arcs); + } + + private void checkIndex(int index) { + if (index < 0 || index >= heads.length) { + throw new IllegalArgumentException("token index out of range: " + index + + ", size: " + heads.length); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencyGraph other)) { + return false; + } + return Arrays.equals(heads, other.heads) && Arrays.equals(relations, other.relations); + } + + @Override + public int hashCode() { + return 31 * Arrays.hashCode(heads) + Arrays.hashCode(relations); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < heads.length; i++) { + if (i > 0) { + sb.append(' '); + } + sb.append(i).append("<-").append(heads[i]).append(':').append(relations[i]); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java new file mode 100644 index 0000000000..0c6e5d1303 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The interface for dependency parsers, which assign every token of a sentence a syntactic + * head and a relation label, forming a single-rooted tree over the sentence. + * + *

Dependency parsing complements the constituency {@link opennlp.tools.parser.Parser}: + * where a constituency parse groups tokens into nested phrases, a dependency parse links + * each token directly to the token it modifies. The result is a {@link DependencyGraph} + * whose indices refer back to the input token array, so spans computed for those tokens + * remain valid for the parse.

+ * + * @see DependencyGraph + * @since 3.0.0 + */ +public interface DependencyParser { + + /** + * Parses a sentence into a {@link DependencyGraph}. + * + * @param tokens The tokens of one sentence. Must not be {@code null} and must contain + * at least one token. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must have the same length as {@code tokens}. + * @return A {@link DependencyGraph} over the given tokens. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokens} or {@code tags} is + * {@code null}, empty, or of mismatched length. + */ + DependencyGraph parse(String[] tokens, String[] tags); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java new file mode 100644 index 0000000000..b805e79a51 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.Arrays; +import java.util.Objects; + +/** + * One dependency-annotated sentence: tokens, their part-of-speech tags, and the gold + * {@link DependencyGraph} over them. Used for training and evaluating a + * {@link DependencyParser}. + * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public class DependencySample { + + private final String[] tokens; + private final String[] tags; + private final DependencyGraph graph; + + /** + * Initializes a {@link DependencySample}. + * + * @param tokens The tokens of the sentence. Must not be {@code null} or empty. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must have the same length as {@code tokens}. + * @param graph The dependency graph over the tokens. Must not be {@code null} and its + * {@link DependencyGraph#size()} must equal the number of tokens. + * @throws IllegalArgumentException Thrown if any parameter is {@code null} or the + * lengths disagree. + */ + public DependencySample(String[] tokens, String[] tags, DependencyGraph graph) { + if (tokens == null || tags == null || graph == null) { + throw new IllegalArgumentException("tokens, tags and graph must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("a sample needs at least one token"); + } + if (tokens.length != tags.length || tokens.length != graph.size()) { + throw new IllegalArgumentException("tokens, tags and graph must agree in length: " + + tokens.length + ", " + tags.length + ", " + graph.size()); + } + this.tokens = tokens.clone(); + this.tags = tags.clone(); + this.graph = graph; + } + + /** + * @return The tokens of the sentence. Never {@code null}. + */ + public String[] getTokens() { + return tokens.clone(); + } + + /** + * @return The part-of-speech tags aligned with the tokens. Never {@code null}. + */ + public String[] getTags() { + return tags.clone(); + } + + /** + * @return The dependency graph over the tokens. Never {@code null}. + */ + public DependencyGraph getGraph() { + return graph; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencySample other)) { + return false; + } + return Arrays.equals(tokens, other.tokens) && Arrays.equals(tags, other.tags) + && graph.equals(other.graph); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(tokens), Arrays.hashCode(tags), graph); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < tokens.length; i++) { + sb.append(i + 1).append('\t').append(tokens[i]).append('\t').append(tags[i]) + .append('\t').append(graph.headOf(i) + 1).append('\t').append(graph.relationOf(i)) + .append(System.lineSeparator()); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java new file mode 100644 index 0000000000..91bf86bd19 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants and accessors of {@link DependencyGraph} and {@link DependencyArc}. + */ +public class DependencyGraphTest { + + private static DependencyGraph sample() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencyGraph graph = sample(); + assertEquals(3, graph.size()); + assertEquals(1, graph.headOf(0)); + assertEquals(2, graph.headOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, graph.headOf(2)); + assertEquals("nsubj", graph.relationOf(1)); + assertEquals(2, graph.root()); + } + + @Test + void testArcsAreInTokenOrder() { + final List arcs = sample().arcs(); + assertEquals(3, arcs.size()); + assertEquals(new DependencyArc(1, 0, "det"), arcs.get(0)); + assertEquals(new DependencyArc(2, 1, "nsubj"), arcs.get(1)); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 2, "root"), arcs.get(2)); + } + + @Test + void testEqualsAndHashCode() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + assertNotEquals(sample(), DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"amod", "nsubj", "root"})); + } + + @Test + void testInputArraysAreCopied() { + final int[] heads = {1, -1}; + final String[] relations = {"nsubj", "root"}; + final DependencyGraph graph = DependencyGraph.of(heads, relations); + heads[0] = 0; + relations[0] = "det"; + assertEquals(1, graph.headOf(0)); + assertEquals("nsubj", graph.relationOf(0)); + } + + @Test + void testNullArraysThrow() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(null, new String[] {"root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, null)); + } + + @Test + void testEmptyGraphThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[0], new String[0])); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, new String[] {"root", "nsubj"})); + } + + @Test + void testRootCountIsEnforced() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, 0}, new String[] {"a", "b"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1, -1}, new String[] {"root", "root"})); + } + + @Test + void testSelfHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {0, -1}, new String[] {"a", "root"})); + } + + @Test + void testOutOfRangeHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {2, -1}, new String[] {"a", "root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-3, -1}, new String[] {"a", "root"})); + } + + @Test + void testBlankRelationThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {" ", "root"})); + } + + @Test + void testIndexBoundsThrow() { + final DependencyGraph graph = sample(); + assertThrows(IllegalArgumentException.class, () -> graph.headOf(-1)); + assertThrows(IllegalArgumentException.class, () -> graph.relationOf(3)); + } + + @Test + void testArcValidation() { + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(0, 0, "root")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, -1, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(-2, 0, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, " ")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java new file mode 100644 index 0000000000..03234eab9f --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants of {@link DependencySample}. + */ +public class DependencySampleTest { + + private static final String[] TOKENS = {"the", "dog", "barks"}; + private static final String[] TAGS = {"DT", "NN", "VBZ"}; + + private static DependencyGraph graph() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencySample sample = new DependencySample(TOKENS, TAGS, graph()); + assertArrayEquals(TOKENS, sample.getTokens()); + assertArrayEquals(TAGS, sample.getTags()); + assertEquals(graph(), sample.getGraph()); + } + + @Test + void testEquals() { + assertEquals(new DependencySample(TOKENS, TAGS, graph()), + new DependencySample(TOKENS, TAGS, graph())); + } + + @Test + void testNullArgumentsThrow() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(null, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, null, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, TAGS, null)); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[] {"one"}, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, new String[] {"DT"}, graph())); + } + + @Test + void testInputArraysAreCopied() { + final String[] tokens = TOKENS.clone(); + final DependencySample sample = new DependencySample(tokens, TAGS, graph()); + tokens[0] = "a"; + assertEquals("the", sample.getTokens()[0]); + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java new file mode 100644 index 0000000000..0dae3f7fc6 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.FilterObjectStream; +import opennlp.tools.util.ObjectStream; + +/** + * Reads {@link DependencySample samples} from a stream of {@link ConlluSentence sentences}, + * mapping the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * + *

Empty nodes carry no basic dependency annotation and are dropped; the remaining word + * lines keep their one-based ids, which are shifted to the zero-based indices of + * {@link DependencyGraph}. Sentences containing a multiword token are skipped entirely: + * {@link ConlluStream} merges such a range with its syntactic words, so the dependency + * annotation of those words is no longer recoverable. Sentences whose annotation is + * incomplete or invalid, for example an underscore head, are skipped as well. Skips are + * counted and the count is logged once the stream is exhausted.

+ * + * @since 3.0.0 + */ +public class ConlluDependencySampleStream + extends FilterObjectStream { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencySampleStream.class); + + private final ConlluTagset tagset; + + private int skipped; + + /** + * Initializes the stream. + * + * @param samples The sentences to convert. Must not be {@code null}. + * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must not + * be {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public ConlluDependencySampleStream(ObjectStream samples, + ConlluTagset tagset) { + super(samples); + if (tagset == null) { + throw new IllegalArgumentException("tagset must not be null"); + } + this.tagset = tagset; + } + + @Override + public DependencySample read() throws IOException { + ConlluSentence sentence; + while ((sentence = samples.read()) != null) { + final DependencySample sample = convert(sentence); + if (sample != null) { + return sample; + } + skipped++; + } + if (skipped > 0) { + logger.warn("Skipped {} sentence(s) without a complete basic dependency annotation.", + skipped); + skipped = 0; + } + return null; + } + + /** + * Converts one sentence, or returns {@code null} when its annotation is unusable. + */ + private DependencySample convert(ConlluSentence sentence) { + final List words = new ArrayList<>(); + for (final ConlluWordLine line : sentence.getWordLines()) { + final String id = line.getId(); + if (id.indexOf('-') >= 0) { + // a merged multiword token: its syntactic words are gone, the sentence is unusable + return null; + } + if (id.indexOf('.') < 0) { + words.add(line); + } + } + if (words.isEmpty()) { + return null; + } + final int n = words.size(); + final String[] tokens = new String[n]; + final String[] tags = new String[n]; + final int[] heads = new int[n]; + final String[] relations = new String[n]; + for (int i = 0; i < n; i++) { + final ConlluWordLine word = words.get(i); + tokens[i] = word.getForm(); + tags[i] = word.getPosTag(tagset); + relations[i] = word.getDeprel(); + try { + heads[i] = Integer.parseInt(word.getHead()) - 1; + } catch (NumberFormatException e) { + return null; + } + } + try { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } catch (IllegalArgumentException e) { + return null; + } + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java new file mode 100644 index 0000000000..6105efd00e --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencySample; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests that {@link ConlluDependencySampleStream} maps the basic dependency columns and + * skips sentences without a usable annotation, including sentences whose multiword tokens + * were merged by {@link ConlluStream}. + */ +public class ConlluDependencySampleStreamTest { + + private static String line(String... fields) { + return String.join("\t", fields); + } + + private static final String CONLLU = String.join("\n", + "# sent_id = test-1", + "# text = He bought the bonds", + line("1", "He", "he", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "bought", "buy", "VERB", "VBD", "_", "0", "root", "_", "_"), + line("3", "the", "the", "DET", "DT", "_", "4", "det", "_", "_"), + line("4", "bonds", "bond", "NOUN", "NNS", "_", "2", "obj", "_", "_"), + "", + "# sent_id = test-2", + "# text = Broken", + line("1", "Broken", "broken", "ADJ", "JJ", "_", "_", "_", "_", "_"), + "", + "# sent_id = test-3", + "# text = im Haus", + line("1-2", "im", "_", "_", "_", "_", "_", "_", "_", "_"), + line("1", "in", "in", "ADP", "APPR", "_", "2", "case", "_", "_"), + line("2", "Haus", "Haus", "NOUN", "NN", "_", "0", "root", "_", "_"), + "", + "# sent_id = test-4", + "# text = Dogs bark", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "") + "\n"; + + private static ConlluDependencySampleStream stream() throws IOException { + return new ConlluDependencySampleStream(new ConlluStream( + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), + ConlluTagset.U); + } + + @Test + void testReadsSamplesAndSkipsUnusableSentences() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + assertArrayEquals(new String[] {"PRON", "VERB", "DET", "NOUN"}, first.getTags()); + assertEquals(1, first.getGraph().headOf(0)); + assertEquals(DependencyArc.ROOT_HEAD, first.getGraph().headOf(1)); + assertEquals(3, first.getGraph().headOf(2)); + assertEquals(1, first.getGraph().headOf(3)); + assertEquals("obj", first.getGraph().relationOf(3)); + + // the underscore-head sentence and the merged-contraction sentence are both skipped + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"Dogs", "bark"}, second.getTokens()); + assertEquals(1, second.getGraph().headOf(0)); + + assertNull(samples.read()); + } + } + + @Test + void testNullTagsetThrows() { + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(new ConlluStream( + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java new file mode 100644 index 0000000000..bd2d05f0ec --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + + +/** + * The static oracle for the arc-standard system: derives the transition sequence that + * reproduces a gold {@link DependencyGraph}. + * + *

An arc is only created once all dependents of the token being attached have been + * collected, which is the arc-standard correctness condition. The oracle is defined for + * projective trees only; a non-projective gold graph has no arc-standard derivation and + * is rejected.

+ * + * @since 3.0.0 + */ +public final class ArcStandardOracle { + + private ArcStandardOracle() { + // static oracle, not meant to be instantiated + } + + /** + * Derives the gold transition sequence for a graph. + * + * @param gold The gold dependency graph. Must not be {@code null} and must be + * projective. + * @return The transitions that rebuild {@code gold} from the start configuration, in + * order. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code gold} is {@code null} or not + * projective. + */ + public static List transitions(DependencyGraph gold) { + if (gold == null) { + throw new IllegalArgumentException("gold must not be null"); + } + final int n = gold.size(); + final int[] goldDependents = new int[n]; + for (int i = 0; i < n; i++) { + final int head = gold.headOf(i); + if (head >= 0) { + goldDependents[head]++; + } + } + + final ArcStandardState state = new ArcStandardState(n); + final List transitions = new ArrayList<>(2 * n); + while (!state.isTerminal()) { + final Transition next = nextTransition(gold, goldDependents, state); + if (next == null) { + throw new IllegalArgumentException( + "gold graph has no arc-standard derivation (non-projective): " + gold); + } + state.apply(next); + transitions.add(next); + } + return transitions; + } + + /** + * Picks the gold transition for the current configuration, or {@code null} when the + * configuration is stuck, which only happens for non-projective input. + */ + private static Transition nextTransition(DependencyGraph gold, int[] goldDependents, + ArcStandardState state) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + if (s1 >= 0 && gold.headOf(s1) == s0) { + final Transition leftArc = Transition.leftArc(gold.relationOf(s1)); + if (state.canApply(leftArc)) { + return leftArc; + } + } + if (s0 >= 0 && s1 != ArcStandardState.NONE && gold.headOf(s0) == s1 + && state.assignedDependents(s0) == goldDependents[s0]) { + final Transition rightArc = Transition.rightArc(gold.relationOf(s0)); + if (state.canApply(rightArc)) { + return rightArc; + } + } + if (state.canApply(Transition.SHIFT)) { + return Transition.SHIFT; + } + return null; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java new file mode 100644 index 0000000000..a261d95f3e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + + +/** + * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining + * tokens, and the arcs assigned so far. + * + *

The stack bottom holds the artificial root, exposed as {@link #ROOT}. Positions that + * do not exist, such as the second stack element in the initial configuration, are exposed + * as {@link #NONE}. A right arc from the artificial root is only applicable once the buffer + * is empty and the root is the only other stack element, which guarantees every completed + * parse has exactly one sentence root.

+ * + *

Instances are confined to a single parse and must not be shared between threads.

+ * + * @since 3.0.0 + */ +public final class ArcStandardState { + + /** The stack value representing the artificial root node. */ + public static final int ROOT = -1; + + /** The value returned for stack or buffer positions that do not exist. */ + public static final int NONE = -2; + + private final int tokenCount; + private final int[] stack; + private final int[] heads; + private final String[] relations; + private final int[] assignedDependents; + + private int top; + private int bufferFront; + + /** + * Initializes the start configuration for a sentence: the artificial root on the stack + * and every token in the buffer. + * + * @param tokenCount The number of tokens in the sentence. Must be greater than zero. + * @throws IllegalArgumentException Thrown if {@code tokenCount} is not positive. + */ + public ArcStandardState(int tokenCount) { + if (tokenCount <= 0) { + throw new IllegalArgumentException("tokenCount must be positive: " + tokenCount); + } + this.tokenCount = tokenCount; + this.stack = new int[tokenCount + 1]; + this.stack[0] = ROOT; + this.top = 0; + this.bufferFront = 0; + this.heads = new int[tokenCount]; + this.relations = new String[tokenCount]; + this.assignedDependents = new int[tokenCount]; + } + + /** + * @return {@code true} if the buffer is empty and only the artificial root remains on + * the stack, so the parse is complete. + */ + public boolean isTerminal() { + return bufferFront == tokenCount && top == 0; + } + + /** + * Checks whether a transition may be applied in the current configuration. + * + * @param transition The transition to check. Must not be {@code null}. + * @return {@code true} if {@link #apply(Transition)} would succeed. + * @throws IllegalArgumentException Thrown if {@code transition} is {@code null}. + */ + public boolean canApply(Transition transition) { + if (transition == null) { + throw new IllegalArgumentException("transition must not be null"); + } + return switch (transition.type()) { + case SHIFT -> bufferFront < tokenCount; + case LEFT_ARC -> top >= 2; + case RIGHT_ARC -> top >= 2 || (top == 1 && bufferFront == tokenCount); + }; + } + + /** + * Applies a transition, updating stack, buffer, and arcs. + * + * @param transition The transition to apply. Must not be {@code null} and must be + * applicable per {@link #canApply(Transition)}. + * @throws IllegalArgumentException Thrown if the transition is {@code null} or not + * applicable in the current configuration. + */ + public void apply(Transition transition) { + if (!canApply(transition)) { + throw new IllegalArgumentException("transition not applicable: " + transition + + " in " + this); + } + switch (transition.type()) { + case SHIFT -> { + top++; + stack[top] = bufferFront++; + } + case LEFT_ARC -> { + final int dependent = stack[top - 1]; + attach(stack[top], dependent, transition.label()); + stack[top - 1] = stack[top]; + top--; + } + case RIGHT_ARC -> { + attach(stack[top - 1], stack[top], transition.label()); + top--; + } + default -> throw new IllegalArgumentException("unsupported type: " + transition.type()); + } + } + + private void attach(int head, int dependent, String relation) { + heads[dependent] = head; + relations[dependent] = relation; + if (head >= 0) { + assignedDependents[head]++; + } + } + + /** + * Retrieves a stack element counted from the top. + * + * @param fromTop Zero for the top element, one for the element below it, and so on. + * Must not be negative. + * @return The token index at that position, {@link #ROOT} for the artificial root, or + * {@link #NONE} if the position does not exist. + * @throws IllegalArgumentException Thrown if {@code fromTop} is negative. + */ + public int stack(int fromTop) { + if (fromTop < 0) { + throw new IllegalArgumentException("fromTop must not be negative: " + fromTop); + } + final int position = top - fromTop; + return position < 0 ? NONE : stack[position]; + } + + /** + * Retrieves a buffer element counted from the front. + * + * @param fromFront Zero for the next token to be shifted, one for the token after it, + * and so on. Must not be negative. + * @return The token index at that position, or {@link #NONE} if the position does not + * exist. + * @throws IllegalArgumentException Thrown if {@code fromFront} is negative. + */ + public int buffer(int fromFront) { + if (fromFront < 0) { + throw new IllegalArgumentException("fromFront must not be negative: " + fromFront); + } + final int position = bufferFront + fromFront; + return position >= tokenCount ? NONE : position; + } + + /** + * @return The number of stack elements including the artificial root. + */ + public int stackSize() { + return top + 1; + } + + /** + * @return The number of tokens still in the buffer. + */ + public int bufferSize() { + return tokenCount - bufferFront; + } + + /** + * Retrieves how many dependents have been attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The number of arcs assigned with the token as head. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int assignedDependents(int index) { + if (index < 0 || index >= tokenCount) { + throw new IllegalArgumentException("token index out of range: " + index); + } + return assignedDependents[index]; + } + + /** + * Builds the {@link DependencyGraph} of a completed parse. + * + * @return The parsed graph. Never {@code null}. + * @throws IllegalStateException Thrown if the parse is not yet terminal. + */ + public DependencyGraph toGraph() { + if (!isTerminal()) { + throw new IllegalStateException("parse is not terminal: " + this); + } + return DependencyGraph.of(heads, relations); + } + + @Override + public String toString() { + return "stackSize=" + stackSize() + ", bufferSize=" + bufferSize() + + ", tokenCount=" + tokenCount; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java new file mode 100644 index 0000000000..90010cbf50 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * Generates the classification features for one arc-standard configuration: words and tags + * of the topmost stack and frontmost buffer positions, their pairings, and a bucketed + * distance between stack top and buffer front. + * + *

Instances hold no state and are safe to share between threads.

+ * + * @since 3.0.0 + */ +public class DependencyContextGenerator { + + private static final String ROOT_VALUE = "*ROOT*"; + private static final String NONE_VALUE = "*NULL*"; + + /** + * Generates the features of the current configuration. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The feature strings. Never {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public String[] getContext(ArcStandardState state, String[] tokens, String[] tags) { + if (state == null || tokens == null || tags == null) { + throw new IllegalArgumentException("state, tokens and tags must not be null"); + } + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int s2 = state.stack(2); + final int b0 = state.buffer(0); + final int b1 = state.buffer(1); + final int b2 = state.buffer(2); + + final String s0w = word(tokens, s0); + final String s0t = tag(tags, s0); + final String s1w = word(tokens, s1); + final String s1t = tag(tags, s1); + final String s2t = tag(tags, s2); + final String b0w = word(tokens, b0); + final String b0t = tag(tags, b0); + final String b1t = tag(tags, b1); + final String b2t = tag(tags, b2); + + final List features = new ArrayList<>(20); + features.add("s0w=" + s0w); + features.add("s0t=" + s0t); + features.add("s1w=" + s1w); + features.add("s1t=" + s1t); + features.add("s2t=" + s2t); + features.add("b0w=" + b0w); + features.add("b0t=" + b0t); + features.add("b1t=" + b1t); + features.add("b2t=" + b2t); + features.add("s0wt=" + s0w + '/' + s0t); + features.add("s0w,b0w=" + s0w + '|' + b0w); + features.add("s0t,b0t=" + s0t + '|' + b0t); + features.add("s1t,s0t=" + s1t + '|' + s0t); + features.add("s1t,s0t,b0t=" + s1t + '|' + s0t + '|' + b0t); + features.add("s0t,b0t,b1t=" + s0t + '|' + b0t + '|' + b1t); + features.add("s0deps=" + dependents(state, s0)); + features.add("s1deps=" + dependents(state, s1)); + features.add("dist=" + distance(s0, b0)); + return features.toArray(new String[0]); + } + + private static String word(String[] tokens, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; + } + + private static String tag(String[] tags, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tags[index]; + } + + private static String dependents(ArcStandardState state, int index) { + return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), 3)); + } + + private static String distance(int s0, int b0) { + if (s0 < 0 || b0 < 0) { + return NONE_VALUE; + } + final int distance = b0 - s0; + return distance >= 4 ? "4+" : Integer.toString(distance); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java new file mode 100644 index 0000000000..f29bada722 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.eval.Evaluator; +import opennlp.tools.util.eval.Mean; + +/** + * Measures the quality of a {@link DependencyParser} against gold + * {@link DependencySample samples} with the two standard scores: the unlabeled attachment + * score (UAS, the fraction of tokens with the correct head) and the labeled attachment + * score (LAS, the fraction of tokens with the correct head and relation label). + * + * @since 3.0.0 + */ +public class DependencyEvaluator extends Evaluator { + + private final DependencyParser parser; + private final Mean uas = new Mean(); + private final Mean las = new Mean(); + + /** + * Initializes a {@link DependencyEvaluator}. + * + * @param parser The parser to evaluate. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyEvaluator(DependencyParser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + /** + * Parses the sample's sentence and scores the prediction against the gold graph. + * + * @param reference The gold sample. Must not be {@code null}. + * @return A {@link DependencySample} carrying the predicted graph. Never {@code null}. + */ + @Override + protected DependencySample processSample(DependencySample reference) { + final DependencyGraph gold = reference.getGraph(); + final DependencyGraph predicted = parser.parse(reference.getTokens(), reference.getTags()); + for (int i = 0; i < gold.size(); i++) { + final boolean headMatches = gold.headOf(i) == predicted.headOf(i); + uas.add(headMatches ? 1 : 0); + las.add(headMatches && gold.relationOf(i).equals(predicted.relationOf(i)) ? 1 : 0); + } + return new DependencySample(reference.getTokens(), reference.getTags(), predicted); + } + + /** + * @return The unlabeled attachment score over all evaluated tokens. + */ + public double getUas() { + return uas.mean(); + } + + /** + * @return The labeled attachment score over all evaluated tokens. + */ + public double getLas() { + return las.mean(); + } + + /** + * @return The number of tokens scored so far. + */ + public long getWordCount() { + return uas.count(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java new file mode 100644 index 0000000000..87fdf27300 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.ml.model.Event; +import opennlp.tools.util.ObjectStream; + +/** + * Turns {@link DependencySample samples} into training {@link Event events}: for each + * sample the {@link ArcStandardOracle} derives the gold transitions, and every transition + * becomes one event pairing the configuration features with the encoded transition. + * + *

Samples whose graph has no arc-standard derivation, that is non-projective trees, + * are skipped and counted; the count is logged once the stream is exhausted.

+ */ +class DependencyEventStream implements ObjectStream { + + private static final Logger logger = LoggerFactory.getLogger(DependencyEventStream.class); + + private final ObjectStream samples; + private final DependencyContextGenerator contextGenerator; + private final Deque pending = new ArrayDeque<>(); + + private int skipped; + + /** + * Initializes the stream. + * + * @param samples The samples to convert. Must not be {@code null}. + * @param contextGenerator The feature generator. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + DependencyEventStream(ObjectStream samples, + DependencyContextGenerator contextGenerator) { + if (samples == null || contextGenerator == null) { + throw new IllegalArgumentException("samples and contextGenerator must not be null"); + } + this.samples = samples; + this.contextGenerator = contextGenerator; + } + + @Override + public Event read() throws IOException { + while (pending.isEmpty()) { + final DependencySample sample = samples.read(); + if (sample == null) { + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + skipped = 0; + } + return null; + } + final List transitions; + try { + transitions = ArcStandardOracle.transitions(sample.getGraph()); + } catch (IllegalArgumentException e) { + skipped++; + continue; + } + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : transitions) { + pending.add(new Event(transition.encode(), + contextGenerator.getContext(state, tokens, tags))); + state.apply(transition); + } + } + return pending.poll(); + } + + @Override + public void reset() throws IOException, UnsupportedOperationException { + samples.reset(); + pending.clear(); + skipped = 0; + } + + @Override + public void close() throws IOException { + samples.close(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java new file mode 100644 index 0000000000..76d8814910 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.HashMap; + +import opennlp.tools.ml.EventTrainer; +import opennlp.tools.ml.TrainerFactory; +import opennlp.tools.ml.TrainerFactory.TrainerType; +import opennlp.tools.ml.model.Event; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.TrainingParameters; + +/** + * A greedy transition-based {@link DependencyParser}: a maximum entropy classifier picks + * the next arc-standard {@link Transition} for each configuration until the parse is + * complete, always taking the highest scoring transition that is applicable. + * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see DependencyParser + * @since 3.0.0 + */ +public class DependencyParserME implements DependencyParser { + + private final MaxentModel model; + private final DependencyContextGenerator contextGenerator; + + /** + * Initializes a {@link DependencyParserME} with a trained transition model. + * + * @param model The transition classification model. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + */ + public DependencyParserME(MaxentModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model; + this.contextGenerator = new DependencyContextGenerator(); + } + + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + state.apply(bestApplicable(state, tokens, tags)); + } + return state.toGraph(); + } + + /** + * Scores all outcomes for the current configuration and picks the best transition that + * is applicable; inapplicable outcomes are passed over regardless of score. + */ + private Transition bestApplicable(ArcStandardState state, String[] tokens, String[] tags) { + final double[] probabilities = model.eval(contextGenerator.getContext(state, tokens, tags)); + Transition best = null; + double bestProbability = Double.NEGATIVE_INFINITY; + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] <= bestProbability) { + continue; + } + final Transition candidate; + try { + candidate = Transition.decode(model.getOutcome(i)); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "model outcome is not a transition: " + model.getOutcome(i), e); + } + if (state.canApply(candidate)) { + best = candidate; + bestProbability = probabilities[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + return best; + } + + /** + * Trains a greedy arc-standard parser from dependency samples. + * + *

Non-projective samples have no arc-standard derivation and are skipped during + * event generation.

+ * + * @param samples The training samples. Must not be {@code null}. + * @param parameters The {@link TrainingParameters}. Must not be {@code null} and must + * select an event model trainer. + * @return A trained {@link DependencyParserME}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or the + * configured trainer is not an event model trainer. + */ + public static DependencyParserME train(ObjectStream samples, + TrainingParameters parameters) throws IOException { + if (samples == null || parameters == null) { + throw new IllegalArgumentException("samples and parameters must not be null"); + } + final TrainerType trainerType = TrainerFactory.getTrainerType(parameters); + if (!TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { + throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); + } + final EventTrainer trainer = + TrainerFactory.getEventTrainer(parameters, new HashMap<>()); + final ObjectStream events = + new DependencyEventStream(samples, new DependencyContextGenerator()); + return new DependencyParserME(trainer.train(events)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java new file mode 100644 index 0000000000..0f9977e675 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * One action of the arc-standard transition system: shift the next buffer token onto the + * stack, or attach one of the two topmost stack tokens to the other under a relation label. + * + *

A transition doubles as a classification outcome: {@link #encode()} renders it as the + * outcome string a model is trained on, and {@link #decode(String)} restores it.

+ * + * @param type The kind of action. Must not be {@code null}. + * @param label The relation label for arc actions, {@code null} for {@link Type#SHIFT}. + * + * @since 3.0.0 + */ +public record Transition(Type type, String label) { + + /** + * The kinds of arc-standard action. + */ + public enum Type { + /** Pushes the front of the buffer onto the stack. */ + SHIFT, + /** Attaches the second stack token to the top one and removes the second. */ + LEFT_ARC, + /** Attaches the top stack token to the second one and removes the top. */ + RIGHT_ARC + } + + /** The single shift transition; shifts carry no label. */ + public static final Transition SHIFT = new Transition(Type.SHIFT, null); + + private static final char SEPARATOR = ':'; + + /** + * Validates the pairing of type and label. + * + * @throws IllegalArgumentException Thrown if {@code type} is {@code null}, a shift + * carries a label, or an arc action has a {@code null} or blank label. + */ + public Transition { + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + if (type == Type.SHIFT) { + if (label != null) { + throw new IllegalArgumentException("a shift must not carry a label: " + label); + } + } else if (label == null || label.isBlank()) { + throw new IllegalArgumentException("an arc transition needs a relation label"); + } + } + + /** + * Creates a left-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#LEFT_ARC}. Never {@code null}. + */ + public static Transition leftArc(String label) { + return new Transition(Type.LEFT_ARC, label); + } + + /** + * Creates a right-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#RIGHT_ARC}. Never {@code null}. + */ + public static Transition rightArc(String label) { + return new Transition(Type.RIGHT_ARC, label); + } + + /** + * Renders the transition as a model outcome string, for example {@code SHIFT} or + * {@code LEFT_ARC:nsubj}. + * + * @return The outcome string. Never {@code null}. + */ + public String encode() { + return type == Type.SHIFT ? type.name() : type.name() + SEPARATOR + label; + } + + /** + * Restores a transition from a model outcome string produced by {@link #encode()}. + * + * @param outcome The outcome string. Must not be {@code null}. + * @return The decoded {@link Transition}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code outcome} is {@code null} or does not + * name a valid transition. + */ + public static Transition decode(String outcome) { + if (outcome == null) { + throw new IllegalArgumentException("outcome must not be null"); + } + if (Type.SHIFT.name().equals(outcome)) { + return SHIFT; + } + final int separator = outcome.indexOf(SEPARATOR); + if (separator < 0) { + throw new IllegalArgumentException("not a transition outcome: " + outcome); + } + final Type type; + try { + type = Type.valueOf(outcome.substring(0, separator)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("not a transition outcome: " + outcome, e); + } + return new Transition(type, outcome.substring(separator + 1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java new file mode 100644 index 0000000000..6cfa9f57ca --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link ArcStandardOracle} derivations replay to the gold graph through + * {@link ArcStandardState}, and that non-projective input is rejected. + */ +public class ArcStandardOracleTest { + + private static DependencyGraph replay(DependencyGraph gold) { + final List transitions = ArcStandardOracle.transitions(gold); + // every token is shifted once and attached once + assertEquals(2 * gold.size(), transitions.size()); + final ArcStandardState state = new ArcStandardState(gold.size()); + for (final Transition transition : transitions) { + assertTrue(state.canApply(transition)); + state.apply(transition); + } + return state.toGraph(); + } + + @Test + void testRoundTripSimpleSentence() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripSingleToken() { + final DependencyGraph gold = DependencyGraph.of(new int[] {-1}, new String[] {"root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripRightBranching() { + // "eat fresh fish now": root with a right dependent that has its own left dependent + final DependencyGraph gold = DependencyGraph.of(new int[] {-1, 2, 0, 0}, + new String[] {"root", "amod", "obj", "advmod"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripDeepChain() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"a", "b", "c", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testNonProjectiveThrows() { + // arcs (2,0) and (3,1) cross, so there is no arc-standard derivation + final DependencyGraph nonProjective = DependencyGraph.of(new int[] {2, 3, -1, 2}, + new String[] {"a", "b", "root", "c"}); + assertThrows(IllegalArgumentException.class, + () -> ArcStandardOracle.transitions(nonProjective)); + } + + @Test + void testNullGraphThrows() { + assertThrows(IllegalArgumentException.class, () -> ArcStandardOracle.transitions(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java new file mode 100644 index 0000000000..def76900fe --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyParserME} end to end: training on a tiny corpus must let the + * greedy parser reproduce the training sentences, which proves the oracle, event stream, + * feature generation, and decode loop agree with each other. + */ +public class DependencyParserMETest { + + private static DependencyParserME parser; + + private static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + private static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + parser = DependencyParserME.train(ObjectStreamUtils.createObjectStream(corpus()), + parameters); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testParseAlwaysYieldsASingleRootedTree() { + // an unseen sentence must still decode to a valid graph, whatever its quality + final DependencyGraph parsed = parser.parse(new String[] {"cats", "sleep"}, + new String[] {"NNS", "VBP"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(320, evaluator.getWordCount()); + } + + @Test + void testParseValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } + + @Test + void testConstructorRejectsNullModel() { + assertThrows(IllegalArgumentException.class, () -> new DependencyParserME(null)); + } + + @Test + void testTrainValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(null, TrainingParameters.defaultParams())); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train( + ObjectStreamUtils.createObjectStream(corpus()), null)); + } +} From c828b8666cd6019a8b0ec441d5b61a7ac72c8987 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 05:23:30 -0400 Subject: [PATCH 34/92] depparse: DependencyModel persistence and a gated Universal Dependencies evaluation Adds DependencyModel on the standard BaseModel machinery, so trained parsers serialize and load like every other tool model; the declared serialVersionUID is the serialver-computed value. Training now returns the model with the trainer's manifest entries, and DependencyParserME gains a model constructor beside the raw one. The round-trip test serializes, reloads, and re-parses. ConlluDependencyParserEvalTest trains on a Universal Dependencies treebank and scores UAS and LAS on its test split; it runs only when opennlp.depparse.ud.dir names the downloaded splits, keeps treebank data out of the repository, and asserts only a low regression floor, with the logged scores as the measurement. (cherry picked from commit c5a3b946b6d8a06a01ff07b961a74ab26a671f3f) --- .../ConlluDependencyParserEvalTest.java | 84 +++++++++++++ .../tools/depparse/DependencyModel.java | 110 ++++++++++++++++++ .../tools/depparse/DependencyParserME.java | 38 ++++-- .../depparse/DependencyParserMETest.java | 37 +++++- 4 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java new file mode 100644 index 0000000000..558547cac8 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.MarkableFileInputStreamFactory; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Trains the greedy arc-standard parser on a Universal Dependencies treebank and scores + * it on the treebank's test split, reporting UAS and LAS. + * + *

Runs only when {@code opennlp.depparse.ud.dir} names a directory containing + * {@code train.conllu} and {@code test.conllu} (a UD treebank's splits, renamed or + * linked). The data is downloaded by the runner and never enters the repository; check + * the treebank's own license before training models for distribution. The assertion is + * a low regression floor; the logged scores are the measurement.

+ */ +public class ConlluDependencyParserEvalTest { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencyParserEvalTest.class); + + @Test + @EnabledIfSystemProperty(named = "opennlp.depparse.ud.dir", matches = ".+") + void testTrainAndScoreOnUniversalDependencies() throws IOException { + final Path dir = Path.of(System.getProperty("opennlp.depparse.ud.dir")); + + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 5); + final long trainStart = System.currentTimeMillis(); + final DependencyModel model; + try (ConlluDependencySampleStream train = samples(dir.resolve("train.conllu"))) { + model = DependencyParserME.train("eng", train, parameters); + } + logger.info("trained in {} ms", System.currentTimeMillis() - trainStart); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new DependencyParserME(model)); + try (ConlluDependencySampleStream test = samples(dir.resolve("test.conllu"))) { + evaluator.evaluate(test); + } + logger.info("UAS {} LAS {} over {} tokens", + evaluator.getUas(), evaluator.getLas(), evaluator.getWordCount()); + + // a regression floor, far below any plausible result; the log line is the measurement + assertTrue(evaluator.getUas() > 0.6d, "UAS regressed below the floor"); + assertTrue(evaluator.getLas() > 0.5d, "LAS regressed below the floor"); + } + + private static ConlluDependencySampleStream samples(Path conllu) throws IOException { + final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); + return new ConlluDependencySampleStream(new ConlluStream(in), ConlluTagset.U); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java new file mode 100644 index 0000000000..8f2c608eeb --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; + +import opennlp.tools.ml.model.AbstractModel; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.model.BaseModel; + +/** + * The persisted form of a trained {@link DependencyParserME}: the transition + * classification model plus the standard model manifest, serialized and loaded through + * the same {@link BaseModel} machinery as every other tool model. + * + * @see DependencyParserME + * @since 3.0.0 + */ +public class DependencyModel extends BaseModel { + + private static final long serialVersionUID = -2928968185269611443L; + + private static final String COMPONENT_NAME = "DependencyParserME"; + static final String PARSER_MODEL_ENTRY_NAME = "depparse.model"; + + /** + * Initializes a {@link DependencyModel} from a trained transition model. + * + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. + * @param parserModel The transition classification model. Must not be {@code null}. + * @param manifestInfoEntries Additional entries for the manifest, or {@code null}. + * @throws IllegalArgumentException Thrown if {@code parserModel} is {@code null}. + */ + public DependencyModel(String languageCode, MaxentModel parserModel, + Map manifestInfoEntries) { + super(COMPONENT_NAME, languageCode, manifestInfoEntries); + if (parserModel == null) { + throw new IllegalArgumentException("parserModel must not be null"); + } + artifactMap.put(PARSER_MODEL_ENTRY_NAME, parserModel); + checkArtifactMap(); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model. + * + * @param in The stream to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(InputStream in) throws IOException { + super(COMPONENT_NAME, in); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelFile The file to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(File modelFile) throws IOException { + super(COMPONENT_NAME, modelFile); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelPath The path to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(Path modelPath) throws IOException { + super(COMPONENT_NAME, modelPath); + } + + @Override + protected void validateArtifactMap() throws InvalidFormatException { + super.validateArtifactMap(); + if (!(artifactMap.get(PARSER_MODEL_ENTRY_NAME) instanceof AbstractModel)) { + throw new InvalidFormatException("The " + PARSER_MODEL_ENTRY_NAME + + " artifact is missing or not a supported transition model."); + } + } + + /** + * @return The transition classification model. Never {@code null}. + */ + public MaxentModel getParserModel() { + return (MaxentModel) artifactMap.get(PARSER_MODEL_ENTRY_NAME); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java index 76d8814910..bbd2d219ff 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Map; import opennlp.tools.ml.EventTrainer; import opennlp.tools.ml.TrainerFactory; @@ -45,7 +46,21 @@ public class DependencyParserME implements DependencyParser { private final DependencyContextGenerator contextGenerator; /** - * Initializes a {@link DependencyParserME} with a trained transition model. + * Initializes a {@link DependencyParserME} from a {@link DependencyModel}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + */ + public DependencyParserME(DependencyModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model.getParserModel(); + this.contextGenerator = new DependencyContextGenerator(); + } + + /** + * Initializes a {@link DependencyParserME} with a raw transition model. * * @param model The transition classification model. Must not be {@code null}. * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. @@ -109,32 +124,37 @@ private Transition bestApplicable(ArcStandardState state, String[] tokens, Strin } /** - * Trains a greedy arc-standard parser from dependency samples. + * Trains a greedy arc-standard parser model from dependency samples. * *

Non-projective samples have no arc-standard derivation and are skipped during * event generation.

* + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. * @param samples The training samples. Must not be {@code null}. * @param parameters The {@link TrainingParameters}. Must not be {@code null} and must * select an event model trainer. - * @return A trained {@link DependencyParserME}. Never {@code null}. + * @return A trained {@link DependencyModel}. Never {@code null}. * @throws IOException Thrown if reading the samples fails. * @throws IllegalArgumentException Thrown if a parameter is {@code null} or the * configured trainer is not an event model trainer. */ - public static DependencyParserME train(ObjectStream samples, - TrainingParameters parameters) throws IOException { - if (samples == null || parameters == null) { - throw new IllegalArgumentException("samples and parameters must not be null"); + public static DependencyModel train(String languageCode, + ObjectStream samples, TrainingParameters parameters) + throws IOException { + if (languageCode == null || samples == null || parameters == null) { + throw new IllegalArgumentException( + "languageCode, samples and parameters must not be null"); } final TrainerType trainerType = TrainerFactory.getTrainerType(parameters); if (!TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } + final Map manifestInfoEntries = new HashMap<>(); final EventTrainer trainer = - TrainerFactory.getEventTrainer(parameters, new HashMap<>()); + TrainerFactory.getEventTrainer(parameters, manifestInfoEntries); final ObjectStream events = new DependencyEventStream(samples, new DependencyContextGenerator()); - return new DependencyParserME(trainer.train(events)); + return new DependencyModel(languageCode, trainer.train(events), manifestInfoEntries); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java index def76900fe..62550ad239 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -38,6 +38,7 @@ */ public class DependencyParserMETest { + private static DependencyModel model; private static DependencyParserME parser; private static DependencySample sample(String[] tokens, String[] tags, int[] heads, @@ -64,8 +65,9 @@ private static List corpus() { static void trainParser() throws IOException { final TrainingParameters parameters = TrainingParameters.defaultParams(); parameters.put(Parameters.CUTOFF_PARAM, 0); - parser = DependencyParserME.train(ObjectStreamUtils.createObjectStream(corpus()), - parameters); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters); + parser = new DependencyParserME(model); } @Test @@ -108,15 +110,40 @@ void testParseValidatesArguments() { @Test void testConstructorRejectsNullModel() { - assertThrows(IllegalArgumentException.class, () -> new DependencyParserME(null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((DependencyModel) null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((opennlp.tools.ml.model.MaxentModel) null)); } @Test void testTrainValidatesArguments() { assertThrows(IllegalArgumentException.class, - () -> DependencyParserME.train(null, TrainingParameters.defaultParams())); + () -> DependencyParserME.train("eng", null, TrainingParameters.defaultParams())); assertThrows(IllegalArgumentException.class, - () -> DependencyParserME.train( + () -> DependencyParserME.train("eng", ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(null, + ObjectStreamUtils.createObjectStream(corpus()), + TrainingParameters.defaultParams())); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + model.serialize(out); + final DependencyModel reloaded = new DependencyModel( + new java.io.ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new DependencyParserME(reloaded) + .parse(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testModelRejectsNullParserModel() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel("eng", null, null)); } } From f7182f63d2ed6daf9bf0df235c82d03437f8050c Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 05:39:22 -0400 Subject: [PATCH 35/92] depparse: Read CoNLL-U raw so multiword-token sentences keep their dependencies Rewrites ConlluDependencySampleStream to parse CoNLL-U content directly instead of consuming the merged ConlluSentence view: ConlluStream merges multiword token ranges with their syntactic words, which suits the token and lemma views but destroys the dependency annotation of every contraction-bearing sentence. The raw reader drops range lines and empty nodes while keeping the syntactic words, so those sentences train and evaluate normally; on the English EWT treebank this recovers 2192 training and 302 test sentences that were previously skipped, and evaluation now covers the full test set. (cherry picked from commit f22b0acb107e52b68b87e56a2d24863df159dd5b) --- .../conllu/ConlluDependencySampleStream.java | 130 ++++++++++++------ .../ConlluDependencyParserEvalTest.java | 2 +- .../ConlluDependencySampleStreamTest.java | 63 +++++++-- 3 files changed, 140 insertions(+), 55 deletions(-) diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 0dae3f7fc6..22f9de9018 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -17,7 +17,10 @@ package opennlp.tools.formats.conllu; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -26,55 +29,68 @@ import opennlp.tools.depparse.DependencyGraph; import opennlp.tools.depparse.DependencySample; -import opennlp.tools.util.FilterObjectStream; +import opennlp.tools.util.InputStreamFactory; import opennlp.tools.util.ObjectStream; /** - * Reads {@link DependencySample samples} from a stream of {@link ConlluSentence sentences}, - * mapping the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * Reads {@link DependencySample samples} directly from CoNLL-U content, mapping the + * {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. * - *

Empty nodes carry no basic dependency annotation and are dropped; the remaining word - * lines keep their one-based ids, which are shifted to the zero-based indices of - * {@link DependencyGraph}. Sentences containing a multiword token are skipped entirely: - * {@link ConlluStream} merges such a range with its syntactic words, so the dependency - * annotation of those words is no longer recoverable. Sentences whose annotation is - * incomplete or invalid, for example an underscore head, are skipped as well. Skips are - * counted and the count is logged once the stream is exhausted.

+ *

The file is parsed raw, deliberately not through {@link ConlluStream}: that stream + * merges multiword token ranges with their syntactic words, which suits the token and + * lemma views but destroys the dependency annotation of every sentence containing a + * contraction. Here range lines and empty nodes are dropped while their syntactic words + * are kept, so contraction-bearing sentences train and evaluate normally. Sentences + * whose annotation is incomplete or invalid, for example an underscore head, are + * skipped and counted; the count is logged once the stream is exhausted.

* * @since 3.0.0 */ -public class ConlluDependencySampleStream - extends FilterObjectStream { +public class ConlluDependencySampleStream implements ObjectStream { private static final Logger logger = LoggerFactory.getLogger(ConlluDependencySampleStream.class); - private final ConlluTagset tagset; + private static final int COLUMNS = 10; + private static final int FORM = 1; + private static final int UPOS = 3; + private static final int XPOS = 4; + private static final int HEAD = 6; + private static final int DEPREL = 7; + private final InputStreamFactory in; + private final int tagColumn; + + private BufferedReader reader; private int skipped; /** * Initializes the stream. * - * @param samples The sentences to convert. Must not be {@code null}. - * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must not - * be {@code null}. + * @param in The CoNLL-U content. Must not be {@code null}. + * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must + * not be {@code null}. + * @throws IOException Thrown if opening the content fails. * @throws IllegalArgumentException Thrown if any parameter is {@code null}. */ - public ConlluDependencySampleStream(ObjectStream samples, - ConlluTagset tagset) { - super(samples); + public ConlluDependencySampleStream(InputStreamFactory in, ConlluTagset tagset) + throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } if (tagset == null) { throw new IllegalArgumentException("tagset must not be null"); } - this.tagset = tagset; + this.in = in; + this.tagColumn = tagset == ConlluTagset.U ? UPOS : XPOS; + this.reader = open(); } @Override public DependencySample read() throws IOException { - ConlluSentence sentence; - while ((sentence = samples.read()) != null) { - final DependencySample sample = convert(sentence); + List words; + while (!(words = nextSentence()).isEmpty()) { + final DependencySample sample = convert(words); if (sample != null) { return sample; } @@ -89,35 +105,50 @@ public DependencySample read() throws IOException { } /** - * Converts one sentence, or returns {@code null} when its annotation is unusable. + * Reads the syntactic word lines of the next sentence: comments, multiword token + * ranges, and empty nodes are dropped; an empty list means the end of the content. */ - private DependencySample convert(ConlluSentence sentence) { - final List words = new ArrayList<>(); - for (final ConlluWordLine line : sentence.getWordLines()) { - final String id = line.getId(); - if (id.indexOf('-') >= 0) { - // a merged multiword token: its syntactic words are gone, the sentence is unusable - return null; + private List nextSentence() throws IOException { + final List words = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + if (line.isBlank()) { + if (!words.isEmpty()) { + return words; + } + continue; } - if (id.indexOf('.') < 0) { - words.add(line); + if (line.charAt(0) == '#') { + continue; + } + final String[] fields = line.split("\t", -1); + if (fields.length < COLUMNS) { + throw new IOException("not a CoNLL-U word line: " + line); + } + final String id = fields[0]; + if (id.indexOf('-') < 0 && id.indexOf('.') < 0) { + words.add(fields); } } - if (words.isEmpty()) { - return null; - } + return words; + } + + /** + * Converts one sentence, or returns {@code null} when its annotation is unusable. + */ + private DependencySample convert(List words) { final int n = words.size(); final String[] tokens = new String[n]; final String[] tags = new String[n]; final int[] heads = new int[n]; final String[] relations = new String[n]; for (int i = 0; i < n; i++) { - final ConlluWordLine word = words.get(i); - tokens[i] = word.getForm(); - tags[i] = word.getPosTag(tagset); - relations[i] = word.getDeprel(); + final String[] word = words.get(i); + tokens[i] = word[FORM]; + tags[i] = word[tagColumn]; + relations[i] = word[DEPREL]; try { - heads[i] = Integer.parseInt(word.getHead()) - 1; + heads[i] = Integer.parseInt(word[HEAD]) - 1; } catch (NumberFormatException e) { return null; } @@ -128,4 +159,21 @@ private DependencySample convert(ConlluSentence sentence) { return null; } } + + @Override + public void reset() throws IOException, UnsupportedOperationException { + reader.close(); + reader = open(); + skipped = 0; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private BufferedReader open() throws IOException { + return new BufferedReader( + new InputStreamReader(in.createInputStream(), StandardCharsets.UTF_8)); + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java index 558547cac8..af4cc1e919 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -79,6 +79,6 @@ void testTrainAndScoreOnUniversalDependencies() throws IOException { private static ConlluDependencySampleStream samples(Path conllu) throws IOException { final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); - return new ConlluDependencySampleStream(new ConlluStream(in), ConlluTagset.U); + return new ConlluDependencySampleStream(in, ConlluTagset.U); } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index 6105efd00e..b7fb44cedd 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -25,6 +25,7 @@ import opennlp.tools.depparse.DependencyArc; import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,9 +34,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; /** - * Tests that {@link ConlluDependencySampleStream} maps the basic dependency columns and - * skips sentences without a usable annotation, including sentences whose multiword tokens - * were merged by {@link ConlluStream}. + * Tests that the raw reader maps the basic dependency columns, keeps the syntactic + * words of multiword tokens while dropping the range line itself, and skips sentences + * without a usable annotation. */ public class ConlluDependencySampleStreamTest { @@ -67,14 +68,16 @@ private static String line(String... fields) { line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), "") + "\n"; + private static InputStreamFactory factory() { + return () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + } + private static ConlluDependencySampleStream stream() throws IOException { - return new ConlluDependencySampleStream(new ConlluStream( - () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), - ConlluTagset.U); + return new ConlluDependencySampleStream(factory(), ConlluTagset.U); } @Test - void testReadsSamplesAndSkipsUnusableSentences() throws IOException { + void testReadsSamplesKeepsContractionsAndSkipsUnusableSentences() throws IOException { try (ConlluDependencySampleStream samples = stream()) { final DependencySample first = samples.read(); assertNotNull(first); @@ -83,23 +86,57 @@ void testReadsSamplesAndSkipsUnusableSentences() throws IOException { assertEquals(1, first.getGraph().headOf(0)); assertEquals(DependencyArc.ROOT_HEAD, first.getGraph().headOf(1)); assertEquals(3, first.getGraph().headOf(2)); - assertEquals(1, first.getGraph().headOf(3)); assertEquals("obj", first.getGraph().relationOf(3)); - // the underscore-head sentence and the merged-contraction sentence are both skipped + // the underscore-head sentence is skipped; the contraction sentence is KEPT, + // with the range line dropped and its syntactic words intact final DependencySample second = samples.read(); assertNotNull(second); - assertArrayEquals(new String[] {"Dogs", "bark"}, second.getTokens()); + assertArrayEquals(new String[] {"in", "Haus"}, second.getTokens()); assertEquals(1, second.getGraph().headOf(0)); + assertEquals("case", second.getGraph().relationOf(0)); + + final DependencySample third = samples.read(); + assertNotNull(third); + assertArrayEquals(new String[] {"Dogs", "bark"}, third.getTokens()); assertNull(samples.read()); } } @Test - void testNullTagsetThrows() { + void testResetRestartsTheStream() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + assertNotNull(samples.read()); + samples.reset(); + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + } + } + + @Test + void testXposTagsetSelectsTheOtherColumn() throws IOException { + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(factory(), ConlluTagset.X)) { + assertArrayEquals(new String[] {"PRP", "VBD", "DT", "NNS"}, + samples.read().getTags()); + } + } + + @Test + void testMalformedLineFailsLoud() { + final InputStreamFactory bad = () -> new ByteArrayInputStream( + "1\ttoo\tfew\tcolumns\n".getBytes(StandardCharsets.UTF_8)); + assertThrows(IOException.class, + () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); + } + + @Test + void testValidation() { + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(null, ConlluTagset.U)); assertThrows(IllegalArgumentException.class, - () -> new ConlluDependencySampleStream(new ConlluStream( - () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), null)); + () -> new ConlluDependencySampleStream(factory(), null)); } } From 14a226a252106fcc622752a24c20f0b1a2765d34 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 05:39:22 -0400 Subject: [PATCH 36/92] depparse: Valency and child features for the transition classifier Extends the configuration features with the partial structure built so far: tags and relations of the leftmost and rightmost dependents of the top two stack tokens, their conjunctions with the neighboring tags, word and tag pairs in both directions, a second-order stack tag triple, buffer word context, and a distance-tag conjunction. ArcStandardState now tracks leftmost and rightmost dependents and exposes the assigned relation of an attached token. Measured together with the raw CoNLL-U reader on UD English EWT with gold UPOS tags: UAS 0.8259 and LAS 0.7929 over the full 25094-token test set, up from 0.7791 and 0.7103 over the 19394-token subset the merged reader could parse; training takes 91 seconds. (cherry picked from commit 6cf87b84bef8f253a57e1cdc9431f46f67642c04) --- .../tools/depparse/ArcStandardState.java | 54 +++++++++++++++- .../depparse/DependencyContextGenerator.java | 63 +++++++++++++++++-- 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index a261d95f3e..0464a3436d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -45,6 +45,8 @@ public final class ArcStandardState { private final int[] heads; private final String[] relations; private final int[] assignedDependents; + private final int[] leftmostDependents; + private final int[] rightmostDependents; private int top; private int bufferFront; @@ -68,6 +70,10 @@ public ArcStandardState(int tokenCount) { this.heads = new int[tokenCount]; this.relations = new String[tokenCount]; this.assignedDependents = new int[tokenCount]; + this.leftmostDependents = new int[tokenCount]; + this.rightmostDependents = new int[tokenCount]; + java.util.Arrays.fill(this.leftmostDependents, NONE); + java.util.Arrays.fill(this.rightmostDependents, NONE); } /** @@ -133,6 +139,12 @@ private void attach(int head, int dependent, String relation) { relations[dependent] = relation; if (head >= 0) { assignedDependents[head]++; + if (leftmostDependents[head] == NONE || dependent < leftmostDependents[head]) { + leftmostDependents[head] = dependent; + } + if (rightmostDependents[head] == NONE || dependent > rightmostDependents[head]) { + rightmostDependents[head] = dependent; + } } } @@ -192,10 +204,50 @@ public int bufferSize() { * @throws IllegalArgumentException Thrown if {@code index} is out of range. */ public int assignedDependents(int index) { + checkTokenIndex(index); + return assignedDependents[index]; + } + + private void checkTokenIndex(int index) { if (index < 0 || index >= tokenCount) { throw new IllegalArgumentException("token index out of range: " + index); } - return assignedDependents[index]; + } + + /** + * Retrieves the leftmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int leftmostDependent(int index) { + checkTokenIndex(index); + return leftmostDependents[index]; + } + + /** + * Retrieves the rightmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int rightmostDependent(int index) { + checkTokenIndex(index); + return rightmostDependents[index]; + } + + /** + * Retrieves the relation a token was attached under, when it has been attached. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The relation label, or {@code null} when the token is still unattached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String assignedRelation(int index) { + checkTokenIndex(index); + return relations[index]; } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java index 90010cbf50..4639bfcbb5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -21,9 +21,10 @@ import java.util.List; /** - * Generates the classification features for one arc-standard configuration: words and tags - * of the topmost stack and frontmost buffer positions, their pairings, and a bucketed - * distance between stack top and buffer front. + * Generates the classification features for one arc-standard configuration: words and + * tags of the topmost stack and frontmost buffer positions, their pairings, the partial + * structure built so far (tags and relations of the leftmost and rightmost dependents, + * valency counts), and a bucketed distance between stack top and buffer front. * *

Instances hold no state and are safe to share between threads.

* @@ -62,10 +63,19 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag final String s2t = tag(tags, s2); final String b0w = word(tokens, b0); final String b0t = tag(tags, b0); + final String b1w = word(tokens, b1); final String b1t = tag(tags, b1); final String b2t = tag(tags, b2); - final List features = new ArrayList<>(20); + final String s0lct = dependentTag(state, tags, s0, true); + final String s0rct = dependentTag(state, tags, s0, false); + final String s1lct = dependentTag(state, tags, s1, true); + final String s1rct = dependentTag(state, tags, s1, false); + final String s0lcl = dependentRelation(state, s0, true); + final String s0rcl = dependentRelation(state, s0, false); + final String s1rcl = dependentRelation(state, s1, false); + + final List features = new ArrayList<>(36); features.add("s0w=" + s0w); features.add("s0t=" + s0t); features.add("s1w=" + s1w); @@ -73,17 +83,36 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag features.add("s2t=" + s2t); features.add("b0w=" + b0w); features.add("b0t=" + b0t); + features.add("b1w=" + b1w); features.add("b1t=" + b1t); features.add("b2t=" + b2t); features.add("s0wt=" + s0w + '/' + s0t); + features.add("s1wt=" + s1w + '/' + s1t); + features.add("b0wt=" + b0w + '/' + b0t); features.add("s0w,b0w=" + s0w + '|' + b0w); features.add("s0t,b0t=" + s0t + '|' + b0t); + features.add("s0w,b0t=" + s0w + '|' + b0t); + features.add("s0t,b0w=" + s0t + '|' + b0w); + features.add("s0wt,b0t=" + s0w + '/' + s0t + '|' + b0t); features.add("s1t,s0t=" + s1t + '|' + s0t); + features.add("s1t,s0w=" + s1t + '|' + s0w); + features.add("s1w,s0t=" + s1w + '|' + s0t); features.add("s1t,s0t,b0t=" + s1t + '|' + s0t + '|' + b0t); features.add("s0t,b0t,b1t=" + s0t + '|' + b0t + '|' + b1t); + features.add("s2t,s1t,s0t=" + s2t + '|' + s1t + '|' + s0t); + features.add("s0lct=" + s0lct); + features.add("s0rct=" + s0rct); + features.add("s1lct=" + s1lct); + features.add("s1rct=" + s1rct); + features.add("s0lcl=" + s0lcl); + features.add("s0rcl=" + s0rcl); + features.add("s1rcl=" + s1rcl); + features.add("s1t,s1rct,s0t=" + s1t + '|' + s1rct + '|' + s0t); + features.add("s0t,s0lct,b0t=" + s0t + '|' + s0lct + '|' + b0t); features.add("s0deps=" + dependents(state, s0)); features.add("s1deps=" + dependents(state, s1)); features.add("dist=" + distance(s0, b0)); + features.add("dist,s0t,b0t=" + distance(s0, b0) + '|' + s0t + '|' + b0t); return features.toArray(new String[0]); } @@ -101,6 +130,32 @@ private static String tag(String[] tags, int index) { return index == ArcStandardState.NONE ? NONE_VALUE : tags[index]; } + /** The tag of a token's leftmost or rightmost dependent attached so far. */ + private static String dependentTag(ArcStandardState state, String[] tags, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + return tag(tags, dependent); + } + + /** The relation of a token's leftmost or rightmost dependent attached so far. */ + private static String dependentRelation(ArcStandardState state, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + if (dependent < 0) { + return NONE_VALUE; + } + final String relation = state.assignedRelation(dependent); + return relation == null ? NONE_VALUE : relation; + } + private static String dependents(ArcStandardState state, int index) { return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), 3)); } From cf9080ba8ab733e9ea250288541dc7efd5302824 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 06:53:14 -0400 Subject: [PATCH 37/92] depparse: Pure-Java feedforward neural tier, training and inference in the JVM Adds the neural transition parser as plain array arithmetic with no native runtime: FeedforwardDependencyModel holds embeddings for words, tags, and arc labels, one cube-activation hidden layer, and a transition output layer in a versioned binary format of our own; FeedforwardDependencyParser decodes greedily with applicability masking; and FeedforwardDependencyTrainer trains the whole network in Java with minibatch AdaGrad over softmax cross-entropy, inverted dropout, a learned unknown word embedding, and a fixed seed for reproducibility. The feature template embeds fourteen configuration positions with second-order children, words and tags for all, labels for the dependent positions. First run on UD English EWT with gold UPOS tags and default settings: UAS 0.8585 and LAS 0.8351 over the full 25094-token test set, above the best classical result (0.8579 and 0.8316 from quasi-Newton training) with 13 minutes of training against 38, parsing at roughly 3.5k tokens per second single-threaded. Untapped levers: pretrained embedding initialization from the static embedding tables, capacity and schedule tuning, and beam search. (cherry picked from commit 7425d11a3ec40c057368be6e5b5dc71b11d6a5ba) --- .../tools/depparse/FeedforwardContext.java | 86 ++++ .../depparse/FeedforwardDependencyModel.java | 327 +++++++++++++ .../depparse/FeedforwardDependencyParser.java | 91 ++++ .../FeedforwardDependencyTrainer.java | 454 ++++++++++++++++++ .../FeedforwardDependencyParserTest.java | 141 ++++++ 5 files changed, 1099 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java new file mode 100644 index 0000000000..10dbcba958 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The feature template of the feedforward parser: a fixed set of configuration + * positions whose words, tags, and arc labels are embedded and concatenated into the + * network input. + * + *

Positions: the top three stack and buffer items; the leftmost and rightmost + * dependents of the top two stack items; and the leftmost dependent of the leftmost + * dependent and rightmost dependent of the rightmost dependent of the top two stack + * items, capturing second-order structure. Words and tags are read for all positions, + * labels only for the dependent positions, whose relations are already assigned.

+ */ +final class FeedforwardContext { + + /** The number of positions whose word and tag are embedded. */ + static final int POSITIONS = 14; + + /** The number of dependent positions whose arc label is embedded. */ + static final int LABEL_POSITIONS = 8; + + private FeedforwardContext() { + // static template only + } + + /** + * Extracts the symbolic features of a configuration: {@link #POSITIONS} words, then + * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent positions + * yield {@code null} entries, which the vocabulary maps to its padding symbol. + */ + static String[] extract(ArcStandardState state, String[] tokens, String[] tags) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int[] positions = { + s0, s1, state.stack(2), + state.buffer(0), state.buffer(1), state.buffer(2), + leftmost(state, s0), rightmost(state, s0), + leftmost(state, s1), rightmost(state, s1), + leftmost(state, leftmost(state, s0)), rightmost(state, rightmost(state, s0)), + leftmost(state, leftmost(state, s1)), rightmost(state, rightmost(state, s1)) + }; + final String[] features = new String[2 * POSITIONS + LABEL_POSITIONS]; + for (int i = 0; i < POSITIONS; i++) { + features[i] = symbol(tokens, positions[i]); + features[POSITIONS + i] = symbol(tags, positions[i]); + } + for (int i = 0; i < LABEL_POSITIONS; i++) { + final int position = positions[6 + i]; + features[2 * POSITIONS + i] = + position >= 0 ? state.assignedRelation(position) : null; + } + return features; + } + + private static int leftmost(ArcStandardState state, int index) { + return index >= 0 ? state.leftmostDependent(index) : ArcStandardState.NONE; + } + + private static int rightmost(ArcStandardState state, int index) { + return index >= 0 ? state.rightmostDependent(index) : ArcStandardState.NONE; + } + + private static String symbol(String[] values, int index) { + if (index == ArcStandardState.ROOT) { + return "*ROOT*"; + } + return index == ArcStandardState.NONE ? null : values[index]; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java new file mode 100644 index 0000000000..e1457efcab --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/** + * The weights of the feedforward transition parser: embeddings for words, tags, and arc + * labels, one hidden layer with cube activation, and a transition output layer, stored + * in a plain versioned binary format with no serialization framework involved. + * + *

This is the pure-Java neural tier: the network is executed with ordinary array + * arithmetic, so parsing needs no native runtime, and the same class scores + * configurations for training and decoding. Unknown words fall back to a learned + * unknown symbol; words are matched case-insensitively. Instances are immutable and + * safe to share between threads.

+ * + * @see FeedforwardDependencyParser + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +public class FeedforwardDependencyModel { + + private static final String MAGIC = "ONLP-FFDP-1"; + + static final String UNKNOWN = "*UNK*"; + static final String ABSENT = "*NULL*"; + + private final Map wordIds; + private final Map tagIds; + private final Map labelIds; + private final String[] transitions; + + private final int embeddingSize; + private final float[][] embeddings; + private final float[][] hiddenWeights; + private final float[] hiddenBias; + private final float[][] outputWeights; + private final float[] outputBias; + + FeedforwardDependencyModel(Map wordIds, Map tagIds, + Map labelIds, String[] transitions, int embeddingSize, + float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias, + float[][] outputWeights, float[] outputBias) { + this.wordIds = wordIds; + this.tagIds = tagIds; + this.labelIds = labelIds; + this.transitions = transitions; + this.embeddingSize = embeddingSize; + this.embeddings = embeddings; + this.hiddenWeights = hiddenWeights; + this.hiddenBias = hiddenBias; + this.outputWeights = outputWeights; + this.outputBias = outputBias; + } + + /** + * Scores every transition for a configuration described by embedding row indices. + * + * @param features The embedding rows of the configuration, as produced by + * {@link #featureIds(String[])}. Must not be {@code null}. + * @return One unnormalized score per transition, indexed like + * {@link #transitions()}. Never {@code null}. + */ + public double[] score(int[] features) { + final int hidden = hiddenBias.length; + final double[] h = new double[hidden]; + for (int j = 0; j < hidden; j++) { + final float[] row = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int f = 0; f < features.length; f++) { + final float[] embedding = embeddings[features[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + sum += row[offset + d] * embedding[d]; + } + } + h[j] = sum * sum * sum; + } + final double[] scores = new double[transitions.length]; + for (int o = 0; o < scores.length; o++) { + final float[] row = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hidden; j++) { + sum += row[j] * h[j]; + } + scores[o] = sum; + } + return scores; + } + + /** + * Maps the symbolic features of {@link FeedforwardContext} onto embedding rows. + * + * @param symbols The symbolic features. Must not be {@code null}. + * @return The embedding row per feature. Never {@code null}. + */ + public int[] featureIds(String[] symbols) { + final int[] ids = new int[symbols.length]; + for (int i = 0; i < FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(wordIds, normalize(symbols[i])); + } + for (int i = FeedforwardContext.POSITIONS; i < 2 * FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(tagIds, symbols[i]); + } + for (int i = 2 * FeedforwardContext.POSITIONS; i < symbols.length; i++) { + ids[i] = lookup(labelIds, symbols[i]); + } + return ids; + } + + /** + * @return The transition outcome strings by output index. Never {@code null}. + */ + public String[] transitions() { + return transitions.clone(); + } + + /** Lowercases a word symbol; special symbols and absences pass through. */ + static String normalize(String word) { + if (word == null) { + return null; + } + return word.startsWith("*") ? word : word.toLowerCase(Locale.ROOT); + } + + private static int lookup(Map ids, String symbol) { + Integer id = ids.get(symbol == null ? ABSENT : symbol); + if (id == null) { + id = ids.get(UNKNOWN); + } + return id; + } + + /** + * Writes the model in the versioned binary format. + * + * @param out The stream to write to. Must not be {@code null}. Not closed. + * @throws IOException Thrown if writing fails. + */ + public void serialize(OutputStream out) throws IOException { + if (out == null) { + throw new IllegalArgumentException("out must not be null"); + } + final DataOutputStream data = new DataOutputStream(new BufferedOutputStream(out)); + data.writeUTF(MAGIC); + writeVocabulary(data, wordIds); + writeVocabulary(data, tagIds); + writeVocabulary(data, labelIds); + data.writeInt(transitions.length); + for (final String transition : transitions) { + data.writeUTF(transition); + } + data.writeInt(embeddingSize); + writeMatrix(data, embeddings); + writeMatrix(data, hiddenWeights); + writeVector(data, hiddenBias); + writeMatrix(data, outputWeights); + writeVector(data, outputBias); + data.flush(); + } + + /** + * Loads a model from the versioned binary format. + * + * @param in The stream to read from. Must not be {@code null}. Not closed. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is not this format. + */ + public static FeedforwardDependencyModel load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final DataInputStream data = new DataInputStream(new BufferedInputStream(in)); + final String magic = data.readUTF(); + if (!MAGIC.equals(magic)) { + throw new IOException("not a feedforward dependency model: " + magic); + } + final Map wordIds = readVocabulary(data); + final Map tagIds = readVocabulary(data); + final Map labelIds = readVocabulary(data); + final String[] transitions = new String[data.readInt()]; + for (int i = 0; i < transitions.length; i++) { + transitions[i] = data.readUTF(); + } + final int embeddingSize = data.readInt(); + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions, + embeddingSize, readMatrix(data), readMatrix(data), readVector(data), + readMatrix(data), readVector(data)); + } + + /** + * Loads a model from a file. + * + * @param path The file to read. Must not be {@code null}. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is not this format. + */ + public static FeedforwardDependencyModel load(Path path) throws IOException { + if (path == null) { + throw new IllegalArgumentException("path must not be null"); + } + try (InputStream in = Files.newInputStream(path)) { + return load(in); + } + } + + private static void writeVocabulary(DataOutputStream data, Map ids) + throws IOException { + data.writeInt(ids.size()); + for (final Map.Entry entry : ids.entrySet()) { + data.writeUTF(entry.getKey()); + data.writeInt(entry.getValue()); + } + } + + private static Map readVocabulary(DataInputStream data) + throws IOException { + final int size = data.readInt(); + final Map ids = new HashMap<>(size * 2); + for (int i = 0; i < size; i++) { + final String symbol = data.readUTF(); + ids.put(symbol, data.readInt()); + } + return ids; + } + + private static void writeMatrix(DataOutputStream data, float[][] matrix) + throws IOException { + data.writeInt(matrix.length); + data.writeInt(matrix.length == 0 ? 0 : matrix[0].length); + for (final float[] row : matrix) { + for (final float value : row) { + data.writeFloat(value); + } + } + } + + private static float[][] readMatrix(DataInputStream data) throws IOException { + final int rows = data.readInt(); + final int columns = data.readInt(); + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = data.readFloat(); + } + } + return matrix; + } + + private static void writeVector(DataOutputStream data, float[] vector) throws IOException { + data.writeInt(vector.length); + for (final float value : vector) { + data.writeFloat(value); + } + } + + private static float[] readVector(DataInputStream data) throws IOException { + final float[] vector = new float[data.readInt()]; + for (int i = 0; i < vector.length; i++) { + vector[i] = data.readFloat(); + } + return vector; + } + + Map wordIds() { + return wordIds; + } + + Map tagIds() { + return tagIds; + } + + Map labelIds() { + return labelIds; + } + + int embeddingSize() { + return embeddingSize; + } + + float[][] embeddings() { + return embeddings; + } + + float[][] hiddenWeights() { + return hiddenWeights; + } + + float[] hiddenBias() { + return hiddenBias; + } + + float[][] outputWeights() { + return outputWeights; + } + + float[] outputBias() { + return outputBias; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java new file mode 100644 index 0000000000..aaae83cdfa --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The pure-Java neural {@link DependencyParser}: a greedy arc-standard decoder over the + * {@link FeedforwardDependencyModel}, picking the highest scoring applicable transition + * for each configuration. + * + *

Inference is ordinary array arithmetic with no native runtime involved, so this + * parser deploys exactly like the classical one while scoring configurations with + * learned dense representations instead of sparse feature conjunctions.

+ * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +public class FeedforwardDependencyParser implements DependencyParser { + + private final FeedforwardDependencyModel model; + private final Transition[] transitions; + + /** + * Initializes a {@link FeedforwardDependencyParser}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model; + final String[] outcomes = model.transitions(); + this.transitions = new Transition[outcomes.length]; + for (int i = 0; i < outcomes.length; i++) { + transitions[i] = Transition.decode(outcomes[i]); + } + } + + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + final double[] scores = model.score( + model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + Transition best = null; + double bestScore = Double.NEGATIVE_INFINITY; + for (int i = 0; i < scores.length; i++) { + if (scores[i] > bestScore && state.canApply(transitions[i])) { + best = transitions[i]; + bestScore = scores[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + state.apply(best); + } + return state.toGraph(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java new file mode 100644 index 0000000000..622e05c05d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.util.ObjectStream; + +/** + * Trains the {@link FeedforwardDependencyModel} entirely in Java: oracle-derived + * transition examples, minibatch AdaGrad over a softmax cross-entropy loss, cube + * activation, and inverted dropout on the hidden layer. No external training framework + * is involved, so the whole neural tier, training and inference, is plain array + * arithmetic inside the JVM. + * + *

Words below the frequency cutoff share a learned unknown embedding; absent + * template positions share a learned padding embedding. Non-projective samples have no + * arc-standard derivation and are skipped. Training is deterministic for a fixed + * {@link Settings#seed()}.

+ * + * @since 3.0.0 + */ +public final class FeedforwardDependencyTrainer { + + private static final Logger logger = + LoggerFactory.getLogger(FeedforwardDependencyTrainer.class); + + private static final double ADAGRAD_EPSILON = 1e-6; + + private FeedforwardDependencyTrainer() { + // static trainer only + } + + /** + * The training hyperparameters. + * + * @param embeddingSize The embedding dimensionality. Must be positive. + * @param hiddenSize The hidden layer width. Must be positive. + * @param epochs The number of passes over the examples. Must be positive. + * @param batchSize The minibatch size. Must be positive. + * @param learningRate The AdaGrad step size. Must be positive. + * @param l2 The L2 penalty applied to the dense weights. Must not be negative. + * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}. + * @param wordCutoff The minimum frequency for a word to get its own embedding. Must + * not be negative. + * @param seed The random seed making a run reproducible. + */ + public record Settings(int embeddingSize, int hiddenSize, int epochs, int batchSize, + double learningRate, double l2, double dropout, int wordCutoff, long seed) { + + /** + * Validates the hyperparameters. + * + * @throws IllegalArgumentException Thrown if a value is outside its documented + * range. + */ + public Settings { + if (embeddingSize <= 0 || hiddenSize <= 0 || epochs <= 0 || batchSize <= 0) { + throw new IllegalArgumentException("sizes, epochs and batch must be positive"); + } + if (learningRate <= 0.0 || l2 < 0.0) { + throw new IllegalArgumentException("learningRate must be positive, l2 not negative"); + } + if (!(dropout >= 0.0 && dropout < 1.0)) { + throw new IllegalArgumentException("dropout must be in [0, 1): " + dropout); + } + if (wordCutoff < 0) { + throw new IllegalArgumentException("wordCutoff must not be negative"); + } + } + + /** + * @return The default hyperparameters. Never {@code null}. + */ + public static Settings defaults() { + return new Settings(50, 200, 10, 256, 0.02, 1e-8, 0.5, 2, 17L); + } + } + + /** + * Trains a model from dependency samples. + * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or no + * trainable example can be derived from the samples. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings) throws IOException { + if (samples == null || settings == null) { + throw new IllegalArgumentException("samples and settings must not be null"); + } + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + final FeedforwardDependencyModel model = initialize(corpus, settings); + final List featureList = new ArrayList<>(); + final List goldList = new ArrayList<>(); + collectExamples(corpus, model, featureList, goldList); + if (featureList.isEmpty()) { + throw new IllegalArgumentException("no trainable examples in the samples"); + } + optimize(model, featureList, goldList, settings); + return model; + } + + /** Builds the vocabularies and randomly initialized weights. */ + private static FeedforwardDependencyModel initialize(List corpus, + Settings settings) { + final Map wordCounts = new HashMap<>(); + final Map tagIds = new HashMap<>(); + final Map labelIds = new HashMap<>(); + final Map transitionIds = new HashMap<>(); + for (final DependencySample s : corpus) { + for (final String token : s.getTokens()) { + wordCounts.merge(FeedforwardDependencyModel.normalize(token), 1, Integer::sum); + } + for (final String tag : s.getTags()) { + tagIds.putIfAbsent(tag, 0); + } + final DependencyGraph graph = s.getGraph(); + for (int i = 0; i < graph.size(); i++) { + labelIds.putIfAbsent(graph.relationOf(i), 0); + } + } + // the outcome space: shift plus both arc directions for every observed label + transitionIds.putIfAbsent(Transition.SHIFT.encode(), 0); + for (final String label : labelIds.keySet()) { + transitionIds.putIfAbsent(Transition.leftArc(label).encode(), 0); + transitionIds.putIfAbsent(Transition.rightArc(label).encode(), 0); + } + + int row = 0; + final Map wordIds = new HashMap<>(); + for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, "*ROOT*")) { + wordIds.put(special, row++); + } + for (final Map.Entry entry : wordCounts.entrySet()) { + if (entry.getValue() >= settings.wordCutoff() && !wordIds.containsKey(entry.getKey())) { + wordIds.put(entry.getKey(), row++); + } + } + final Map tags = new HashMap<>(); + for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, "*ROOT*")) { + tags.put(special, row++); + } + for (final String tag : tagIds.keySet()) { + tags.put(tag, row++); + } + final Map labels = new HashMap<>(); + for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT)) { + labels.put(special, row++); + } + for (final String label : labelIds.keySet()) { + labels.put(label, row++); + } + + int transitionIndex = 0; + final String[] transitions = new String[transitionIds.size()]; + for (final String encoded : transitionIds.keySet()) { + transitions[transitionIndex] = encoded; + transitionIds.put(encoded, transitionIndex++); + } + + final Random random = new Random(settings.seed()); + final int inputSize = + (2 * FeedforwardContext.POSITIONS + FeedforwardContext.LABEL_POSITIONS) + * settings.embeddingSize(); + final float[][] embeddings = uniform(random, row, settings.embeddingSize(), 0.01); + final float[][] hiddenWeights = uniform(random, settings.hiddenSize(), inputSize, + Math.sqrt(6.0 / (inputSize + settings.hiddenSize()))); + final float[][] outputWeights = uniform(random, transitions.length, + settings.hiddenSize(), + Math.sqrt(6.0 / (settings.hiddenSize() + transitions.length))); + return new FeedforwardDependencyModel(wordIds, tags, labels, transitions, + settings.embeddingSize(), embeddings, hiddenWeights, + new float[settings.hiddenSize()], outputWeights, new float[transitions.length]); + } + + /** Replays the oracle over every projective sample, emitting one example per step. */ + private static void collectExamples(List corpus, + FeedforwardDependencyModel model, List featureList, List goldList) { + final Map transitionIds = new HashMap<>(); + final String[] transitions = model.transitions(); + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(transitions[i], i); + } + int skipped = 0; + for (final DependencySample sample : corpus) { + final List oracle; + try { + oracle = ArcStandardOracle.transitions(sample.getGraph()); + } catch (IllegalArgumentException e) { + skipped++; + continue; + } + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : oracle) { + featureList.add(model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + goldList.add(transitionIds.get(transition.encode())); + state.apply(transition); + } + } + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + } + } + + /** Minibatch AdaGrad over softmax cross-entropy with cube activation and dropout. */ + private static void optimize(FeedforwardDependencyModel model, List featureList, + List goldList, Settings settings) { + final int exampleCount = featureList.size(); + final int[][] features = featureList.toArray(new int[0][]); + final int[] gold = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + gold[i] = goldList.get(i); + } + + final float[][] embeddings = model.embeddings(); + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + final int embeddingSize = settings.embeddingSize(); + final int hiddenSize = settings.hiddenSize(); + final int outputSize = outputBias.length; + final int inputSize = features[0].length * embeddingSize; + + final double[][] embeddingAccumulator = + new double[embeddings.length][embeddingSize]; + final double[][] hiddenAccumulator = new double[hiddenSize][inputSize]; + final double[] hiddenBiasAccumulator = new double[hiddenSize]; + final double[][] outputAccumulator = new double[outputSize][hiddenSize]; + final double[] outputBiasAccumulator = new double[outputSize]; + + final double[][] hiddenGradient = new double[hiddenSize][inputSize]; + final double[] hiddenBiasGradient = new double[hiddenSize]; + final double[][] outputGradient = new double[outputSize][hiddenSize]; + final double[] outputBiasGradient = new double[outputSize]; + final Map embeddingGradients = new HashMap<>(); + + final Random random = new Random(settings.seed()); + final int[] order = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + order[i] = i; + } + + final double keep = 1.0 - settings.dropout(); + final double[] x = new double[inputSize]; + final double[] pre = new double[hiddenSize]; + final double[] hidden = new double[hiddenSize]; + final boolean[] mask = new boolean[hiddenSize]; + final double[] probabilities = new double[outputSize]; + final double[] hiddenDelta = new double[hiddenSize]; + final double[] inputDelta = new double[inputSize]; + + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + for (int batchStart = 0; batchStart < exampleCount; + batchStart += settings.batchSize()) { + final int batchEnd = Math.min(batchStart + settings.batchSize(), exampleCount); + final int batch = batchEnd - batchStart; + zero(hiddenGradient); + java.util.Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + java.util.Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + + for (int b = batchStart; b < batchEnd; b++) { + final int[] feats = features[order[b]]; + final int goldTransition = gold[order[b]]; + for (int f = 0; f < feats.length; f++) { + final float[] embedding = embeddings[feats[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + for (int j = 0; j < hiddenSize; j++) { + mask[j] = random.nextDouble() < keep; + if (!mask[j]) { + pre[j] = 0.0; + hidden[j] = 0.0; + continue; + } + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum / keep; + } + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + max = Math.max(max, sum); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + for (int o = 0; o < outputSize; o++) { + probabilities[o] /= normalizer; + } + loss -= Math.log(Math.max(probabilities[goldTransition], 1e-12)); + + java.util.Arrays.fill(hiddenDelta, 0.0); + java.util.Arrays.fill(inputDelta, 0.0); + for (int o = 0; o < outputSize; o++) { + final double delta = probabilities[o] - (o == goldTransition ? 1.0 : 0.0); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + for (int j = 0; j < hiddenSize; j++) { + if (!mask[j]) { + continue; + } + final double preDelta = hiddenDelta[j] * 3.0 * pre[j] * pre[j] / keep; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < feats.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(feats[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + update(hiddenWeights, hiddenGradient, hiddenAccumulator, batch, settings); + updateVector(hiddenBias, hiddenBiasGradient, hiddenBiasAccumulator, batch, settings); + update(outputWeights, outputGradient, outputAccumulator, batch, settings); + updateVector(outputBias, outputBiasGradient, outputBiasAccumulator, batch, settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = embeddings[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d] / batch; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + } + logger.info("epoch {}: loss {} in {} ms", epoch, loss / exampleCount, + System.currentTimeMillis() - epochStart); + } + } + + private static void update(float[][] weights, double[][] gradients, + double[][] accumulators, int batch, Settings settings) { + for (int r = 0; r < weights.length; r++) { + final float[] weightRow = weights[r]; + final double[] gradientRow = gradients[r]; + final double[] accumulatorRow = accumulators[r]; + for (int c = 0; c < weightRow.length; c++) { + final double gradient = gradientRow[c] / batch + settings.l2() * weightRow[c]; + accumulatorRow[c] += gradient * gradient; + weightRow[c] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[c]) + ADAGRAD_EPSILON); + } + } + } + + private static void updateVector(float[] weights, double[] gradients, + double[] accumulators, int batch, Settings settings) { + for (int i = 0; i < weights.length; i++) { + final double gradient = gradients[i] / batch; + accumulators[i] += gradient * gradient; + weights[i] -= settings.learningRate() * gradient + / (Math.sqrt(accumulators[i]) + ADAGRAD_EPSILON); + } + } + + private static float[][] uniform(Random random, int rows, int columns, double scale) { + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = (float) ((random.nextDouble() * 2.0 - 1.0) * scale); + } + } + return matrix; + } + + private static void zero(double[][] matrix) { + for (final double[] row : matrix) { + java.util.Arrays.fill(row, 0.0); + } + } + + private static void shuffle(int[] order, Random random) { + for (int i = order.length - 1; i > 0; i--) { + final int j = random.nextInt(i + 1); + final int swap = order[i]; + order[i] = order[j]; + order[j] = swap; + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java new file mode 100644 index 0000000000..253c55c48d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStreamUtils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the pure-Java neural tier end to end: training on a tiny corpus must let the + * greedy feedforward parser reproduce the training sentences, and a model must survive + * the serialization round trip bit-for-bit in behavior. + */ +public class FeedforwardDependencyParserTest { + + private static FeedforwardDependencyModel model; + private static FeedforwardDependencyParser parser; + + private static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + private static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + @BeforeAll + static void trainParser() throws IOException { + // dropout off so the tiny network memorizes deterministically + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 120, 32, 0.05, 0.0, 0.0, 1, 17L); + model = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + parser = new FeedforwardDependencyParser(model); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + @Test + void testUnknownWordsStillYieldASingleRootedTree() { + final DependencyGraph parsed = parser.parse(new String[] {"unseen", "words"}, + new String[] {"JJ", "NNS"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + final FeedforwardDependencyModel reloaded = + FeedforwardDependencyModel.load(new ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new FeedforwardDependencyParser(reloaded) + .parse(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), parsed); + } + + @Test + void testCorruptModelFailsLoud() { + assertThrows(IOException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream("not a model".getBytes()))); + } + + @Test + void testSettingsValidation() { + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(0, 32, 10, 32, 0.05, 0.0, 0.0, 1, 17L)); + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(16, 32, 10, 32, -1.0, 0.0, 0.0, 1, 17L)); + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(16, 32, 10, 32, 0.05, 0.0, 1.0, 1, 17L)); + } + + @Test + void testArgumentValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null)); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train(null, + FeedforwardDependencyTrainer.Settings.defaults())); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } +} From 944cff4d0428e27408a80f48be51e272959e6ab5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 06:58:34 -0400 Subject: [PATCH 38/92] depparse: Pretrained word embedding seeding for the feedforward trainer Adds a training overload taking a pretrained vector provider: vocabulary words the provider knows start from their pretrained vectors instead of random noise, stay trainable, and everything the provider does not know keeps the random initialization. The provider is a training-time ingredient only, since the learned embeddings ship inside the model, so parsing carries no dependency on the embedding source. This is the seam that lets the static embedding tables feed the parser. (cherry picked from commit 0084cf36f6fb74a634fe6d0e422c743f2180aa15) --- .../FeedforwardDependencyTrainer.java | 54 +++++++++++++++++++ .../FeedforwardDependencyParserTest.java | 19 +++++++ 2 files changed, 73 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index 622e05c05d..f0ae0d4b91 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -112,6 +112,33 @@ public static Settings defaults() { */ public static FeedforwardDependencyModel train(ObjectStream samples, Settings settings) throws IOException { + return train(samples, settings, null); + } + + /** + * Trains a model from dependency samples, seeding word embeddings from a pretrained + * source. + * + *

The provider is consulted once per vocabulary word during initialization; words + * it returns {@code null} for keep their random initialization, and all embeddings + * remain trainable afterwards. The pretrained source is a training-time ingredient + * only: the learned embeddings ship inside the model, so parsing carries no + * dependency on the source.

+ * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @param pretrained Maps a normalized word to its pretrained vector of exactly + * {@link Settings#embeddingSize()} dimensions, or {@code null} for + * unknown words. May be {@code null} to disable seeding. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if {@code samples} or {@code settings} is + * {@code null}, no trainable example can be derived, or a pretrained vector + * has the wrong dimensionality. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings, java.util.function.Function pretrained) + throws IOException { if (samples == null || settings == null) { throw new IllegalArgumentException("samples and settings must not be null"); } @@ -121,6 +148,9 @@ public static FeedforwardDependencyModel train(ObjectStream sa corpus.add(sample); } final FeedforwardDependencyModel model = initialize(corpus, settings); + if (pretrained != null) { + seed(model, pretrained, settings); + } final List featureList = new ArrayList<>(); final List goldList = new ArrayList<>(); collectExamples(corpus, model, featureList, goldList); @@ -131,6 +161,30 @@ public static FeedforwardDependencyModel train(ObjectStream sa return model; } + /** Overwrites the random word rows with pretrained vectors where available. */ + private static void seed(FeedforwardDependencyModel model, + java.util.function.Function pretrained, Settings settings) { + int seeded = 0; + for (final Map.Entry entry : model.wordIds().entrySet()) { + if (entry.getKey().startsWith("*")) { + continue; // the special symbols have no pretrained meaning + } + final float[] vector = pretrained.apply(entry.getKey()); + if (vector == null) { + continue; + } + if (vector.length != settings.embeddingSize()) { + throw new IllegalArgumentException("pretrained vector for '" + entry.getKey() + + "' has " + vector.length + " dimensions, expected " + + settings.embeddingSize()); + } + System.arraycopy(vector, 0, model.embeddings()[entry.getValue()], 0, vector.length); + seeded++; + } + logger.info("seeded {} of {} word embeddings from the pretrained source", seeded, + model.wordIds().size()); + } + /** Builds the vocabularies and randomly initialized weights. */ private static FeedforwardDependencyModel initialize(List corpus, Settings settings) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 253c55c48d..6f762faeb4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -123,6 +123,25 @@ void testSettingsValidation() { .Settings(16, 32, 10, 32, 0.05, 0.0, 1.0, 1, 17L)); } + @Test + void testPretrainedSeedingAppliesAndValidates() throws IOException { + // near-zero learning keeps the seeded row observable after one epoch + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(4, 8, 1, 32, 1e-9, 0.0, 0.0, 1, 17L); + final float[] vector = {0.25f, -0.5f, 0.75f, -1.0f}; + final FeedforwardDependencyModel seeded = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> "dog".equals(word) ? vector.clone() : null); + final int row = seeded.wordIds().get("dog"); + for (int d = 0; d < vector.length; d++) { + assertEquals(vector[d], seeded.embeddings()[row][d], 1e-4); + } + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> new float[] {1.0f})); + } + @Test void testArgumentValidation() { assertThrows(IllegalArgumentException.class, From ecfdc9effebd62a305ab555ecd87c2546355c30e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 09:34:26 -0400 Subject: [PATCH 39/92] depparse: Beam search decoding for the feedforward parser The decoder keeps the best transition sequences side by side, scored by summed log-probabilities, so one locally attractive but globally wrong transition no longer commits the whole parse. Beam size one keeps the exact greedy fast path; every arc-standard derivation has the same length, so summed scores compare without normalization. (cherry picked from commit 2c3009983cf075f8f77c3ae88224368a68195dab) --- .../tools/depparse/ArcStandardState.java | 22 +++ .../depparse/FeedforwardDependencyParser.java | 133 +++++++++++++++++- .../FeedforwardDependencyParserTest.java | 37 +++++ 3 files changed, 188 insertions(+), 4 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index 0464a3436d..8f79eeff1f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -76,6 +76,28 @@ public ArcStandardState(int tokenCount) { java.util.Arrays.fill(this.rightmostDependents, NONE); } + private ArcStandardState(ArcStandardState source) { + this.tokenCount = source.tokenCount; + this.stack = source.stack.clone(); + this.heads = source.heads.clone(); + this.relations = source.relations.clone(); + this.assignedDependents = source.assignedDependents.clone(); + this.leftmostDependents = source.leftmostDependents.clone(); + this.rightmostDependents = source.rightmostDependents.clone(); + this.top = source.top; + this.bufferFront = source.bufferFront; + } + + /** + * Creates an independent copy of this configuration, so alternatives can be advanced + * side by side during search. + * + * @return A copy that can be advanced without affecting this state. Never {@code null}. + */ + public ArcStandardState copy() { + return new ArcStandardState(this); + } + /** * @return {@code true} if the buffer is empty and only the artificial root remains on * the stack, so the parse is complete. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index aaae83cdfa..0ada7c985a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -17,10 +17,20 @@ package opennlp.tools.depparse; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + /** - * The pure-Java neural {@link DependencyParser}: a greedy arc-standard decoder over the - * {@link FeedforwardDependencyModel}, picking the highest scoring applicable transition - * for each configuration. + * The pure-Java neural {@link DependencyParser}: an arc-standard decoder over the + * {@link FeedforwardDependencyModel}, greedy by default and beamed when constructed + * with a beam size above one. + * + *

With a beam, the decoder keeps the highest scoring transition sequences side by + * side, scored by summed log-probabilities, so one locally attractive but globally + * wrong transition no longer commits the whole parse. Every complete arc-standard + * derivation of a sentence has the same length, which keeps the summed scores + * comparable without length normalization.

* *

Inference is ordinary array arithmetic with no native runtime involved, so this * parser deploys exactly like the classical one while scoring configurations with @@ -36,19 +46,38 @@ public class FeedforwardDependencyParser implements DependencyParser { private final FeedforwardDependencyModel model; private final Transition[] transitions; + private final int beamSize; /** - * Initializes a {@link FeedforwardDependencyParser}. + * Initializes a greedy {@link FeedforwardDependencyParser}. * * @param model The model to parse with. Must not be {@code null}. * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an * outcome of the model does not decode to a transition. */ public FeedforwardDependencyParser(FeedforwardDependencyModel model) { + this(model, 1); + } + + /** + * Initializes a {@link FeedforwardDependencyParser} with a beam. + * + * @param model The model to parse with. Must not be {@code null}. + * @param beamSize The number of transition sequences to keep side by side. Must be + * greater than zero; {@code 1} decodes greedily. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}, + * {@code beamSize} is not positive, or an outcome of the model does not + * decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model, int beamSize) { if (model == null) { throw new IllegalArgumentException("model must not be null"); } + if (beamSize < 1) { + throw new IllegalArgumentException("beamSize must be positive: " + beamSize); + } this.model = model; + this.beamSize = beamSize; final String[] outcomes = model.transitions(); this.transitions = new Transition[outcomes.length]; for (int i = 0; i < outcomes.length; i++) { @@ -68,6 +97,20 @@ public DependencyGraph parse(String[] tokens, String[] tags) { throw new IllegalArgumentException("tokens and tags must have the same length: " + tokens.length + " != " + tags.length); } + if (beamSize == 1) { + return greedyParse(tokens, tags); + } + return beamParse(tokens, tags); + } + + /** + * Decodes greedily: the highest scoring applicable transition wins each step. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + */ + private DependencyGraph greedyParse(String[] tokens, String[] tags) { final ArcStandardState state = new ArcStandardState(tokens.length); while (!state.isTerminal()) { final double[] scores = model.score( @@ -88,4 +131,86 @@ public DependencyGraph parse(String[] tokens, String[] tags) { } return state.toGraph(); } + + /** One search alternative: a configuration, its summed log-probability score, and the + * transition that would advance it, {@code null} once complete. */ + private record Alternative(ArcStandardState state, double score, Transition next) { + } + + /** + * Decodes with a beam: the {@code beamSize} best transition sequences advance side by + * side and the best complete one wins. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + */ + private DependencyGraph beamParse(String[] tokens, String[] tags) { + List beam = + List.of(new Alternative(new ArcStandardState(tokens.length), 0.0, null)); + while (true) { + boolean advanced = false; + final List expansions = new ArrayList<>(); + for (final Alternative alternative : beam) { + if (alternative.state().isTerminal()) { + expansions.add(alternative); + continue; + } + advanced = true; + final double[] logProbabilities = logSoftmax(model.score( + model.featureIds(FeedforwardContext.extract(alternative.state(), tokens, tags)))); + for (int i = 0; i < logProbabilities.length; i++) { + if (alternative.state().canApply(transitions[i])) { + expansions.add(new Alternative(alternative.state(), + alternative.score() + logProbabilities[i], transitions[i])); + } + } + } + if (!advanced) { + break; + } + expansions.sort(Comparator.comparingDouble(Alternative::score).reversed()); + final List survivors = + new ArrayList<>(Math.min(beamSize, expansions.size())); + for (int i = 0; i < expansions.size() && survivors.size() < beamSize; i++) { + final Alternative expansion = expansions.get(i); + if (expansion.next() == null) { + survivors.add(expansion); + } else { + final ArcStandardState state = expansion.state().copy(); + state.apply(expansion.next()); + survivors.add(new Alternative(state, expansion.score(), null)); + } + } + if (survivors.isEmpty()) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in the beam"); + } + beam = survivors; + } + return beam.get(0).state().toGraph(); + } + + /** + * Normalizes raw transition scores to log-probabilities. + * + * @param scores The raw output scores. + * @return The log-softmax of {@code scores}. Never {@code null}. + */ + private static double[] logSoftmax(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = max + Math.log(sum); + final double[] logProbabilities = new double[scores.length]; + for (int i = 0; i < scores.length; i++) { + logProbabilities[i] = scores[i] - logSum; + } + return logProbabilities; + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 6f762faeb4..48bf25cfa3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -95,6 +95,43 @@ void testUnknownWordsStillYieldASingleRootedTree() { parsed.root(); } + @Test + void testBeamOfOneMatchesGreedy() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 1); + for (final DependencySample sample : corpus()) { + assertEquals(parser.parse(sample.getTokens(), sample.getTags()), + beamed.parse(sample.getTokens(), sample.getTags())); + } + } + + @Test + void testBeamedParserReproducesTrainingSentences() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 4); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + beamed.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + @Test + void testBeamedParseIsDeterministicAndSingleRooted() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 8); + final String[] tokens = {"unseen", "words", "everywhere"}; + final String[] tags = {"JJ", "NNS", "RB"}; + final DependencyGraph first = beamed.parse(tokens, tags); + assertEquals(first, beamed.parse(tokens, tags)); + assertEquals(3, first.size()); + first.root(); + } + + @Test + void testBeamSizeValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(model, 0)); + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null, 4)); + } + @Test void testModelRoundTripThroughSerialization() throws IOException { final ByteArrayOutputStream out = new ByteArrayOutputStream(); From 63cd289fc86ac78773e8f081c7ad3f36d16367cd Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 10:33:46 -0400 Subject: [PATCH 40/92] depparse: Global refinement with beam search and early update After local training, sentences are re-decoded with a beam while the gold derivation is tracked through it; the moment gold falls out an early update pushes the model toward keeping it, under a conditional likelihood over the beam's candidate paths. Paths are scored exactly as the beamed parser scores them, so training optimizes the quantity decoding uses. Refinement mutates the model in place, deterministic for a fixed seed. (cherry picked from commit bd663ba9cb577ffb91e1e2ec89619bb5ad60c7ed) --- .../FeedforwardDependencyTrainer.java | 386 ++++++++++++++++++ .../FeedforwardDependencyParserTest.java | 28 ++ 2 files changed, 414 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index f0ae0d4b91..c676949ca9 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -161,6 +161,392 @@ public static FeedforwardDependencyModel train(ObjectStream sa return model; } + /** + * Fine-tunes a locally trained model globally: sentences are decoded with a beam, the + * gold derivation is tracked through it, and the moment the gold prefix falls out of + * the beam an early update pushes the model toward keeping it. The loss is a + * conditional likelihood over the beam's candidate paths, scored exactly like the + * beamed parser scores them, summed log-probabilities, so training optimizes the + * quantity decoding uses. + * + *

The model is updated in place with per-sentence AdaGrad steps and no dropout; + * {@link Settings#epochs()} counts the refinement passes. Refinement is deterministic + * for a fixed {@link Settings#seed()}. Parse afterwards with the same beam size.

+ * + * @param model The locally trained model to refine. Must not be {@code null}. + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters; {@code epochs}, {@code learningRate}, + * {@code l2}, and {@code seed} apply. Must not be {@code null}. + * @param beamSize The beam width to track the gold derivation in. Must be at least 2. + * @return The same model instance, refined. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code beamSize} is below 2, or no trainable sample can be derived. + */ + public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model, + ObjectStream samples, Settings settings, int beamSize) + throws IOException { + if (model == null || samples == null || settings == null) { + throw new IllegalArgumentException("model, samples and settings must not be null"); + } + if (beamSize < 2) { + throw new IllegalArgumentException("beamSize must be at least 2: " + beamSize); + } + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + final Map transitionIds = new HashMap<>(); + final Transition[] transitions = new Transition[model.transitions().length]; + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(model.transitions()[i], i); + transitions[i] = Transition.decode(model.transitions()[i]); + } + + final List trainable = new ArrayList<>(); + final List oracles = new ArrayList<>(); + for (final DependencySample s : corpus) { + final List oracle; + try { + oracle = ArcStandardOracle.transitions(s.getGraph()); + } catch (IllegalArgumentException e) { + continue; + } + final int[] encoded = new int[oracle.size()]; + for (int i = 0; i < encoded.length; i++) { + encoded[i] = transitionIds.get(oracle.get(i).encode()); + } + trainable.add(s); + oracles.add(encoded); + } + if (trainable.isEmpty()) { + throw new IllegalArgumentException("no trainable samples for refinement"); + } + + final GlobalOptimizer optimizer = new GlobalOptimizer(model, settings); + final Random random = new Random(settings.seed()); + final int[] order = new int[trainable.size()]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + int updates = 0; + for (final int index : order) { + final double sentenceLoss = optimizer.refineSentence(trainable.get(index), + oracles.get(index), transitions, beamSize); + if (sentenceLoss >= 0.0) { + loss += sentenceLoss; + updates++; + } + } + logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch, + loss / Math.max(updates, 1), updates, System.currentTimeMillis() - epochStart); + } + return model; + } + + /** One candidate path in the refinement beam: the parent link forms the history. */ + private static final class BeamNode { + private final BeamNode parent; + private final int[] features; + private final int transition; + private final double score; + private final boolean gold; + private ArcStandardState state; + + private BeamNode(BeamNode parent, int[] features, int transition, double score, + boolean gold) { + this.parent = parent; + this.features = features; + this.transition = transition; + this.score = score; + this.gold = gold; + } + } + + /** The forward, backward, and AdaGrad state for global refinement. */ + private static final class GlobalOptimizer { + private final FeedforwardDependencyModel model; + private final Settings settings; + private final int embeddingSize; + private final int hiddenSize; + private final int outputSize; + private final int inputSize; + + private final double[][] embeddingAccumulator; + private final double[][] hiddenAccumulator; + private final double[] hiddenBiasAccumulator; + private final double[][] outputAccumulator; + private final double[] outputBiasAccumulator; + + private final double[][] hiddenGradient; + private final double[] hiddenBiasGradient; + private final double[][] outputGradient; + private final double[] outputBiasGradient; + private final Map embeddingGradients = new HashMap<>(); + + private final double[] x; + private final double[] pre; + private final double[] hidden; + private final double[] probabilities; + private final double[] hiddenDelta; + private final double[] inputDelta; + + private GlobalOptimizer(FeedforwardDependencyModel model, Settings settings) { + this.model = model; + this.settings = settings; + this.embeddingSize = model.embeddings()[0].length; + this.hiddenSize = model.hiddenBias().length; + this.outputSize = model.outputBias().length; + this.inputSize = + (2 * FeedforwardContext.POSITIONS + FeedforwardContext.LABEL_POSITIONS) + * embeddingSize; + this.embeddingAccumulator = + new double[model.embeddings().length][embeddingSize]; + this.hiddenAccumulator = new double[hiddenSize][inputSize]; + this.hiddenBiasAccumulator = new double[hiddenSize]; + this.outputAccumulator = new double[outputSize][hiddenSize]; + this.outputBiasAccumulator = new double[outputSize]; + this.hiddenGradient = new double[hiddenSize][inputSize]; + this.hiddenBiasGradient = new double[hiddenSize]; + this.outputGradient = new double[outputSize][hiddenSize]; + this.outputBiasGradient = new double[outputSize]; + this.x = new double[inputSize]; + this.pre = new double[hiddenSize]; + this.hidden = new double[hiddenSize]; + this.probabilities = new double[outputSize]; + this.hiddenDelta = new double[hiddenSize]; + this.inputDelta = new double[inputSize]; + } + + /** + * Decodes one sentence with the beam, updating on the early-update point or the + * final beam. + * + * @param sample The sentence. + * @param oracle The gold transition indexes. + * @param transitions The decoded transition inventory. + * @param beamSize The beam width. + * @return The sentence loss, or {@code -1} when the sentence produced no update. + */ + private double refineSentence(DependencySample sample, int[] oracle, + Transition[] transitions, int beamSize) { + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + final BeamNode root = new BeamNode(null, null, -1, 0.0, true); + root.state = new ArcStandardState(tokens.length); + List beam = List.of(root); + + for (int step = 0; step < oracle.length; step++) { + final List expansions = new ArrayList<>(); + BeamNode goldChild = null; + for (final BeamNode node : beam) { + final int[] features = + model.featureIds(FeedforwardContext.extract(node.state, tokens, tags)); + forward(features); + logSoftmaxInPlace(probabilities); + for (int i = 0; i < outputSize; i++) { + if (node.state.canApply(transitions[i])) { + final boolean goldNext = node.gold && i == oracle[step]; + final BeamNode child = new BeamNode(node, features, i, + node.score + probabilities[i], goldNext); + expansions.add(child); + if (goldNext) { + goldChild = child; + } + } + } + } + expansions.sort((a, b) -> Double.compare(b.score, a.score)); + final List survivors = + new ArrayList<>(expansions.subList(0, Math.min(beamSize, expansions.size()))); + boolean goldSurvives = false; + for (final BeamNode survivor : survivors) { + if (survivor.gold) { + goldSurvives = true; + break; + } + } + if (!goldSurvives) { + if (goldChild == null) { + return -1.0; // the oracle transition was inapplicable; nothing to learn from + } + survivors.add(goldChild); + return updateFromCandidates(survivors); + } + if (step == oracle.length - 1) { + return updateFromCandidates(survivors); + } + for (final BeamNode survivor : survivors) { + survivor.state = survivor.parent.state.copy(); + survivor.state.apply(transitions[survivor.transition]); + } + beam = survivors; + } + return -1.0; + } + + /** Applies the conditional-likelihood update over the candidate paths. */ + private double updateFromCandidates(List candidates) { + double max = Double.NEGATIVE_INFINITY; + double goldScore = Double.NEGATIVE_INFINITY; + for (final BeamNode candidate : candidates) { + max = Math.max(max, candidate.score); + if (candidate.gold) { + goldScore = candidate.score; + } + } + double normalizer = 0.0; + for (final BeamNode candidate : candidates) { + normalizer += Math.exp(candidate.score - max); + } + final double logNormalizer = max + Math.log(normalizer); + + zero(hiddenGradient); + java.util.Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + java.util.Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + for (final BeamNode candidate : candidates) { + final double weight = Math.exp(candidate.score - logNormalizer) + - (candidate.gold ? 1.0 : 0.0); + if (weight == 0.0) { + continue; + } + for (BeamNode node = candidate; node.parent != null; node = node.parent) { + backward(node.features, node.transition, weight); + } + } + update(model.hiddenWeights(), hiddenGradient, hiddenAccumulator, 1, settings); + updateVector(model.hiddenBias(), hiddenBiasGradient, hiddenBiasAccumulator, 1, + settings); + update(model.outputWeights(), outputGradient, outputAccumulator, 1, settings); + updateVector(model.outputBias(), outputBiasGradient, outputBiasAccumulator, 1, + settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = model.embeddings()[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d]; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + return logNormalizer - goldScore; + } + + /** Computes hidden activations and raw output scores for one feature vector. */ + private void forward(int[] features) { + final float[][] embeddings = model.embeddings(); + for (int f = 0; f < features.length; f++) { + final float[] embedding = embeddings[features[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + for (int j = 0; j < hiddenSize; j++) { + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum; + } + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + } + } + + /** + * Accumulates gradients for one decoded step: the weighted difference between the + * step's softmax and its chosen transition. + * + * @param features The step's input features. + * @param chosen The transition the path took at this step. + * @param weight The path's weight in the candidate distribution. + */ + private void backward(int[] features, int chosen, double weight) { + forward(features); + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + max = Math.max(max, probabilities[o]); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + java.util.Arrays.fill(hiddenDelta, 0.0); + java.util.Arrays.fill(inputDelta, 0.0); + final float[][] outputWeights = model.outputWeights(); + for (int o = 0; o < outputSize; o++) { + // dL/dlogit for a path's step under the conditional likelihood: the path weight + // times how the step's log-probability responds to this logit + final double delta = + weight * ((o == chosen ? 1.0 : 0.0) - probabilities[o] / normalizer); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + for (int j = 0; j < hiddenSize; j++) { + final double preDelta = hiddenDelta[j] * 3.0 * pre[j] * pre[j]; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < features.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(features[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + /** Turns raw scores into log-probabilities in place. */ + private static void logSoftmaxInPlace(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = max + Math.log(sum); + for (int i = 0; i < scores.length; i++) { + scores[i] -= logSum; + } + } + } + /** Overwrites the random word rows with pretrained vectors where available. */ private static void seed(FeedforwardDependencyModel model, java.util.function.Function pretrained, Settings settings) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 48bf25cfa3..a4a8c347d8 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -124,6 +124,34 @@ void testBeamedParseIsDeterministicAndSingleRooted() { first.root(); } + @Test + void testRefinementKeepsToyPerformance() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final FeedforwardDependencyTrainer.Settings refineSettings = + new FeedforwardDependencyTrainer.Settings(16, 32, 2, 32, 0.01, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new FeedforwardDependencyParser(refined, 2)); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + @Test + void testRefineValidation() { + final FeedforwardDependencyTrainer.Settings settings = + FeedforwardDependencyTrainer.Settings.defaults(); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(null, ObjectStreamUtils.createObjectStream(corpus()), settings, 4)); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(model, ObjectStreamUtils.createObjectStream(corpus()), settings, 1)); + } + @Test void testBeamSizeValidation() { assertThrows(IllegalArgumentException.class, From 273499018295ccf21f9f99cd453609e36ac14064 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:21:13 -0400 Subject: [PATCH 41/92] depparse: State, transition, usage, and edge-case tests; expanded trainer and parser javadoc --- .../ConlluDependencyParserUsageTest.java | 178 ++++++++++++ .../ConlluDependencySampleStreamTest.java | 35 +++ .../tools/depparse/ArcStandardOracle.java | 2 +- .../tools/depparse/FeedforwardContext.java | 2 +- .../depparse/FeedforwardDependencyParser.java | 8 +- .../FeedforwardDependencyTrainer.java | 13 +- .../tools/depparse/ArcStandardStateTest.java | 148 ++++++++++ .../DependencyParserEdgeCaseTest.java | 264 ++++++++++++++++++ .../tools/depparse/TransitionTest.java | 73 +++++ 9 files changed, 713 insertions(+), 10 deletions(-) create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java new file mode 100644 index 0000000000..960653a778 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Demonstrates the full dependency parsing workflow on a self-contained fixture: read + * gold sentences from CoNLL-U content, train a {@link DependencyParserME}, parse a + * sentence, inspect the resulting {@link DependencyGraph}, and persist the model. + * + *

The fixture holds three tiny sentences inline, so the test needs no external data. + * A real treebank provides thousands of sentences; here each fixture sentence is + * repeated to give the trainer the same evidence many times, which lets the model + * memorize the fixture and makes every expected value exact.

+ */ +public class ConlluDependencyParserUsageTest { + + /** + * Joins the ten CoNLL-U columns of one word line with tabs. + * + * @param fields The column values; exactly ten are expected by the format. + * @return The joined word line. Never {@code null}. + */ + private static String line(String... fields) { + return String.join("\t", fields); + } + + /** + * The training fixture: three gold sentences in CoNLL-U form. The {@code HEAD} column + * is one-based with {@code 0} marking the root; the reader converts it to the + * zero-based convention of {@link DependencyGraph}. + */ + private static final String CONLLU = String.join("\n", + "# text = the dog barks", + line("1", "the", "the", "DET", "DT", "_", "2", "det", "_", "_"), + line("2", "dog", "dog", "NOUN", "NN", "_", "3", "nsubj", "_", "_"), + line("3", "barks", "bark", "VERB", "VBZ", "_", "0", "root", "_", "_"), + "", + "# text = dogs bark", + line("1", "dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "", + "# text = she eats fish", + line("1", "she", "she", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "eats", "eat", "VERB", "VBZ", "_", "0", "root", "_", "_"), + line("3", "fish", "fish", "NOUN", "NN", "_", "2", "obj", "_", "_"), + "") + "\n"; + + private static DependencyModel model; + private static DependencyParserME parser; + + /** + * Reads the fixture sentences through the CoNLL-U reader. + * + * @return One sample per fixture sentence, in file order. Never {@code null}. + * @throws IOException Thrown if reading the in-memory content fails. + */ + private static List readFixture() throws IOException { + final InputStreamFactory in = + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + final List samples = new ArrayList<>(); + try (ConlluDependencySampleStream stream = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + DependencySample sample; + while ((sample = stream.read()) != null) { + samples.add(sample); + } + } + return samples; + } + + /** + * Trains the parser once for all tests: read the fixture, repeat it for evidence, + * and hand the samples to the trainer. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final List fixture = readFixture(); + final List trainingSamples = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + trainingSamples.addAll(fixture); + } + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(trainingSamples), parameters); + parser = new DependencyParserME(model); + } + + @Test + void testReaderDeliversTheGoldAnnotation() throws IOException { + final List fixture = readFixture(); + assertEquals(3, fixture.size()); + final DependencySample first = fixture.get(0); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), first.getGraph()); + assertEquals("NOUN", first.getTags()[1]); + } + + @Test + void testParseAssignsHeadsAndRelations() { + // Parsing takes the tokens and their part-of-speech tags; the result names, for + // every token, its head token and the relation between the two. + final DependencyGraph parse = parser.parse( + new String[] {"the", "dog", "barks"}, new String[] {"DET", "NOUN", "VERB"}); + assertEquals(1, parse.headOf(0)); + assertEquals("det", parse.relationOf(0)); + assertEquals(2, parse.headOf(1)); + assertEquals("nsubj", parse.relationOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, parse.headOf(2)); + assertEquals("root", parse.relationOf(2)); + assertEquals(2, parse.root()); + } + + @Test + void testEvaluatorScoresTheParserAgainstGoldSamples() throws IOException { + // The evaluator parses each gold sentence and accumulates the two standard scores; + // on its own training fixture the memorizing model is exact on all eight tokens. + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(readFixture())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(8, evaluator.getWordCount()); + } + + @Test + void testPersistedModelParsesLikeTheOriginal(@TempDir Path dir) throws IOException { + // A trained model is saved to a file and loaded back like any other tool model; the + // reloaded parser must produce the exact same parse as the original. + final Path file = dir.resolve("en-depparse.bin"); + model.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRON", "VERB", "NOUN"})); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index b7fb44cedd..f0a1a40740 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -132,6 +132,41 @@ void testMalformedLineFailsLoud() { () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); } + @Test + void testSemanticallyInvalidAnnotationIsSkippedNotFatal() throws IOException { + // Structurally well-formed lines whose annotation cannot form a valid tree, here an + // out-of-range head and a rootless cycle, skip the sentence instead of failing, so + // one broken sentence cannot abort reading a large treebank. + final String content = String.join("\n", + line("1", "far", "far", "ADV", "RB", "_", "5", "advmod", "_", "_"), + line("2", "off", "off", "ADP", "RP", "_", "0", "root", "_", "_"), + "", + line("1", "loop", "loop", "NOUN", "NN", "_", "2", "dep", "_", "_"), + line("2", "back", "back", "ADV", "RB", "_", "1", "dep", "_", "_"), + "", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample onlyValid = samples.read(); + assertNotNull(onlyValid); + assertArrayEquals(new String[] {"Fine"}, onlyValid.getTokens()); + assertEquals(DependencyArc.ROOT_HEAD, onlyValid.getGraph().headOf(0)); + assertNull(samples.read()); + } + } + + @Test + void testEmptyContentYieldsNoSample() throws IOException { + final InputStreamFactory in = () -> new ByteArrayInputStream(new byte[0]); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + assertNull(samples.read()); + } + } + @Test void testValidation() { assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java index bd2d05f0ec..596ef4d2d4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java @@ -35,7 +35,7 @@ public final class ArcStandardOracle { private ArcStandardOracle() { - // static oracle, not meant to be instantiated + // This class only exposes static derivation methods and is never instantiated. } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java index 10dbcba958..8778071c77 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -37,7 +37,7 @@ final class FeedforwardContext { static final int LABEL_POSITIONS = 8; private FeedforwardContext() { - // static template only + // This class only exposes the static feature template and is never instantiated. } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index 0ada7c985a..aa5b64f622 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -27,10 +27,10 @@ * with a beam size above one. * *

With a beam, the decoder keeps the highest scoring transition sequences side by - * side, scored by summed log-probabilities, so one locally attractive but globally - * wrong transition no longer commits the whole parse. Every complete arc-standard - * derivation of a sentence has the same length, which keeps the summed scores - * comparable without length normalization.

+ * side, scored by summed log-probabilities, so a single locally attractive but globally + * wrong transition can still be recovered while the correct parse remains inside the + * beam. Every complete arc-standard derivation of a sentence has the same length, which + * keeps the summed scores comparable without length normalization.

* *

Inference is ordinary array arithmetic with no native runtime involved, so this * parser deploys exactly like the classical one while scoring configurations with diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index c676949ca9..deab7611d6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -51,7 +51,7 @@ public final class FeedforwardDependencyTrainer { private static final double ADAGRAD_EPSILON = 1e-6; private FeedforwardDependencyTrainer() { - // static trainer only + // This class only exposes static training methods and is never instantiated. } /** @@ -373,7 +373,9 @@ private double refineSentence(DependencySample sample, int[] oracle, } if (!goldSurvives) { if (goldChild == null) { - return -1.0; // the oracle transition was inapplicable; nothing to learn from + // The gold transition was not applicable in the gold configuration, so + // this sentence yields no update. + return -1.0; } survivors.add(goldChild); return updateFromCandidates(survivors); @@ -553,7 +555,9 @@ private static void seed(FeedforwardDependencyModel model, int seeded = 0; for (final Map.Entry entry : model.wordIds().entrySet()) { if (entry.getKey().startsWith("*")) { - continue; // the special symbols have no pretrained meaning + // The special unknown, padding, and root symbols have no pretrained + // counterpart, so they keep their random initialization. + continue; } final float[] vector = pretrained.apply(entry.getKey()); if (vector == null) { @@ -590,7 +594,8 @@ private static FeedforwardDependencyModel initialize(List corp labelIds.putIfAbsent(graph.relationOf(i), 0); } } - // the outcome space: shift plus both arc directions for every observed label + // The outcome space is the shift transition plus both arc directions for every + // relation label observed in the training data. transitionIds.putIfAbsent(Transition.SHIFT.encode(), 0); for (final String label : labelIds.keySet()) { transitionIds.putIfAbsent(Transition.leftArc(label).encode(), 0); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java new file mode 100644 index 0000000000..9cc6cc0b47 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the configuration mechanics of {@link ArcStandardState}: the start + * configuration, transition applicability at the boundaries, the bookkeeping of attached + * dependents, copy independence, and the fail-loud behavior of every accessor. + */ +public class ArcStandardStateTest { + + @Test + void testInitialConfiguration() { + final ArcStandardState state = new ArcStandardState(3); + assertEquals(ArcStandardState.ROOT, state.stack(0)); + assertEquals(ArcStandardState.NONE, state.stack(1)); + assertEquals(0, state.buffer(0)); + assertEquals(1, state.buffer(1)); + assertEquals(2, state.buffer(2)); + assertEquals(ArcStandardState.NONE, state.buffer(3)); + assertEquals(1, state.stackSize()); + assertEquals(3, state.bufferSize()); + assertFalse(state.isTerminal()); + } + + @Test + void testSingleTokenDerivationIsForced() { + // With one token the system permits exactly one derivation: shift the token, then + // attach it to the artificial root with a right arc. + final ArcStandardState state = new ArcStandardState(1); + assertTrue(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertFalse(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.SHIFT); + assertFalse(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertTrue(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + state.toGraph()); + } + + @Test + void testDependentBookkeepingDuringADerivation() { + // Derives "the dog barks" (the<-dog via det, dog<-barks via nsubj, barks<-root) and + // checks the partial-structure accessors after every attachment. + final ArcStandardState state = new ArcStandardState(3); + state.apply(Transition.SHIFT); + state.apply(Transition.SHIFT); + assertEquals(0, state.assignedDependents(1)); + assertNull(state.assignedRelation(0)); + + state.apply(Transition.leftArc("det")); + assertEquals(1, state.assignedDependents(1)); + assertEquals(0, state.leftmostDependent(1)); + assertEquals(0, state.rightmostDependent(1)); + assertEquals("det", state.assignedRelation(0)); + + state.apply(Transition.SHIFT); + state.apply(Transition.leftArc("nsubj")); + assertEquals(1, state.leftmostDependent(2)); + assertEquals("nsubj", state.assignedRelation(1)); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), state.toGraph()); + } + + @Test + void testInapplicableTransitionFailsLoud() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.leftArc("det"))); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.rightArc("root"))); + assertThrows(IllegalArgumentException.class, () -> state.apply(null)); + assertThrows(IllegalArgumentException.class, () -> state.canApply(null)); + } + + @Test + void testToGraphBeforeTerminalFailsLoud() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalStateException.class, state::toGraph); + state.apply(Transition.SHIFT); + assertThrows(IllegalStateException.class, state::toGraph); + } + + @Test + void testCopyIsIndependentOfTheOriginal() { + final ArcStandardState original = new ArcStandardState(2); + original.apply(Transition.SHIFT); + final ArcStandardState copy = original.copy(); + + copy.apply(Transition.SHIFT); + copy.apply(Transition.leftArc("nsubj")); + // The copy advanced by two transitions while the original still has one token + // buffered and one on the stack. + assertEquals(2, original.stackSize()); + assertEquals(1, original.bufferSize()); + assertEquals(0, original.assignedDependents(1)); + assertEquals(1, copy.assignedDependents(1)); + } + + @Test + void testAccessorValidation() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, () -> state.stack(-1)); + assertThrows(IllegalArgumentException.class, () -> state.buffer(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(2)); + assertThrows(IllegalArgumentException.class, () -> state.leftmostDependent(2)); + assertThrows(IllegalArgumentException.class, () -> state.rightmostDependent(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedRelation(2)); + } + + @Test + void testTokenCountMustBePositive() { + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(-1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java new file mode 100644 index 0000000000..f7015089e9 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the boundary behavior of both dependency parsers: empty and single-token input, + * non-projective sentences during training and parsing, and the file round trip of both + * model formats, which must reproduce the exact parses of the original models. + */ +public class DependencyParserEdgeCaseTest { + + private static DependencyModel maxentModel; + private static DependencyParserME maxentParser; + private static FeedforwardDependencyModel feedforwardModel; + private static FeedforwardDependencyParser feedforwardParser; + + /** + * Builds one gold sample from its parallel arrays. + * + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @param heads The zero-based head per token, {@code -1} for the root. + * @param relations The relation label per token. + * @return The assembled sample. Never {@code null}. + */ + private static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + /** + * Builds the projective training corpus: three tiny sentences, each repeated so both + * trainers see enough evidence to memorize them. + * + * @return The training samples. Never {@code null}. + */ + private static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + /** + * Builds a four-token sample whose gold arcs (2,0) and (3,1) cross, so the tree is + * non-projective and has no arc-standard derivation. + * + * @return The non-projective sample. Never {@code null}. + */ + private static DependencySample nonProjectiveSample() { + return sample(new String[] {"the", "dog", "barks", "today"}, + new String[] {"DT", "NN", "VBZ", "RB"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "nsubj", "root", "advmod"}); + } + + /** + * Trains one classical and one neural model on the shared corpus. The feedforward + * settings disable dropout and fix the seed, so the tiny network memorizes the corpus + * deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParsers() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + maxentModel = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters); + maxentParser = new DependencyParserME(maxentModel); + + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + feedforwardModel = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + feedforwardParser = new FeedforwardDependencyParser(feedforwardModel); + } + + @Test + void testEmptySentenceIsRejectedByBothParsers() { + assertThrows(IllegalArgumentException.class, + () -> maxentParser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> feedforwardParser.parse(new String[0], new String[0])); + // The transition system itself has no configuration for zero tokens either. + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + } + + @Test + void testSingleTokenSentenceAttachesToTheRoot() { + // A single token permits only the derivation shift then right-arc, so the head is + // forced to the artificial root and the model only chooses the relation label. + final DependencyGraph maxentParse = + maxentParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), maxentParse); + + final DependencyGraph feedforwardParse = + feedforwardParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + feedforwardParse); + } + + @Test + void testNonProjectiveSamplesAreSkippedDuringTraining() throws IOException { + // One non-projective sample joins the corpus; it cannot yield events, so training + // proceeds on the remaining samples and still memorizes the projective sentences. + final List mixed = new ArrayList<>(corpus()); + mixed.add(nonProjectiveSample()); + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + final DependencyModel model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(mixed), parameters); + final DependencyParserME parser = new DependencyParserME(model); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + parser.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"})); + } + + @Test + void testNonProjectiveGoldDecodesToAProjectiveTree() { + // The parser can only emit arc-standard derivations, so for a sentence whose gold + // tree is non-projective the prediction is necessarily a different, projective tree. + final DependencySample gold = nonProjectiveSample(); + final DependencyGraph parsed = maxentParser.parse(gold.getTokens(), gold.getTags()); + assertNotEquals(gold.getGraph(), parsed); + assertEquals(0, crossingArcCount(parsed)); + // The unseen final token becomes the root and the verb attaches under it; the + // familiar determiner and subject arcs survive from the training evidence. + assertEquals(DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"det", "nsubj", "nsubj", "root"}), parsed); + } + + @Test + void testFeedforwardTrainingFailsLoudWithoutProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(8, 8, 1, 32, 0.05, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings)); + assertEquals("no trainable examples in the samples", e.getMessage()); + } + + @Test + void testRefinementFailsLoudWithoutProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(feedforwardModel, + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings, 2)); + assertEquals("no trainable samples for refinement", e.getMessage()); + } + + @Test + void testMaxentModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse.bin"); + maxentModel.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + for (final DependencySample sample : corpus()) { + assertEquals(maxentParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags())); + } + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + reloaded.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + @Test + void testFeedforwardModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse-ff.bin"); + try (OutputStream out = Files.newOutputStream(file)) { + feedforwardModel.serialize(out); + } + final FeedforwardDependencyParser reloaded = + new FeedforwardDependencyParser(FeedforwardDependencyModel.load(file)); + for (final DependencySample sample : corpus()) { + assertEquals(feedforwardParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags())); + } + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"})); + } + + /** + * Counts the pairs of crossing arcs in a graph, treating the root arc as spanning + * from a virtual position left of the sentence to its dependent. A projective tree + * has zero crossing pairs. + * + * @param graph The graph to inspect. Must not be {@code null}. + * @return The number of crossing arc pairs. + * @throws IllegalArgumentException Thrown if {@code graph} is {@code null}. + */ + private static int crossingArcCount(DependencyGraph graph) { + if (graph == null) { + throw new IllegalArgumentException("graph must not be null"); + } + int crossings = 0; + for (int i = 0; i < graph.size(); i++) { + for (int j = i + 1; j < graph.size(); j++) { + final int iLow = Math.min(i, graph.headOf(i)); + final int iHigh = Math.max(i, graph.headOf(i)); + final int jLow = Math.min(j, graph.headOf(j)); + final int jHigh = Math.max(j, graph.headOf(j)); + if ((iLow < jLow && jLow < iHigh && iHigh < jHigh) + || (jLow < iLow && iLow < jHigh && jHigh < iHigh)) { + crossings++; + } + } + } + return crossings; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java new file mode 100644 index 0000000000..a46866855c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the outcome encoding of {@link Transition}: every transition must render to a + * unique outcome string and decode back to an equal transition, and every malformed + * outcome must be rejected loudly. + */ +public class TransitionTest { + + @Test + void testEncodeRendersTheOutcomeStrings() { + assertEquals("SHIFT", Transition.SHIFT.encode()); + assertEquals("LEFT_ARC:nsubj", Transition.leftArc("nsubj").encode()); + assertEquals("RIGHT_ARC:obj", Transition.rightArc("obj").encode()); + } + + @Test + void testDecodeRestoresEncodedTransitions() { + assertSame(Transition.SHIFT, Transition.decode("SHIFT")); + assertEquals(Transition.leftArc("nsubj"), Transition.decode("LEFT_ARC:nsubj")); + assertEquals(Transition.rightArc("obj"), Transition.decode("RIGHT_ARC:obj")); + } + + @Test + void testLabelContainingTheSeparatorRoundTrips() { + // Only the first separator splits type from label, so a label containing the + // separator character itself survives the round trip unchanged. + final Transition transition = Transition.leftArc("nmod:poss"); + assertEquals("LEFT_ARC:nmod:poss", transition.encode()); + assertEquals(transition, Transition.decode(transition.encode())); + } + + @Test + void testDecodeRejectsMalformedOutcomes() { + assertThrows(IllegalArgumentException.class, () -> Transition.decode(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN")); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN:det")); + // A labeled shift is contradictory and must be rejected by the record invariant. + assertThrows(IllegalArgumentException.class, () -> Transition.decode("SHIFT:det")); + } + + @Test + void testConstructorValidation() { + assertThrows(IllegalArgumentException.class, () -> new Transition(null, "det")); + assertThrows(IllegalArgumentException.class, + () -> new Transition(Transition.Type.SHIFT, "det")); + assertThrows(IllegalArgumentException.class, () -> Transition.leftArc(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.rightArc(" ")); + } +} From 752c05c5c7891dc237be02250f5f6737b3adf1a9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:35:47 -0400 Subject: [PATCH 42/92] depparse: Document treebank acquisition for the gated evaluation with a download helper --- .../dev/README-ud-treebanks.md | 48 ++++++++++++++ .../dev/download-ud-treebank.sh | 66 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 opennlp-core/opennlp-formats/dev/README-ud-treebanks.md create mode 100755 opennlp-core/opennlp-formats/dev/download-ud-treebank.sh diff --git a/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md new file mode 100644 index 0000000000..3052d88553 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md @@ -0,0 +1,48 @@ + + +# Universal Dependencies treebanks for the dependency parser evaluation + +The dependency parser's unit tests are fully self-contained, but its accuracy evaluation runs against a real Universal Dependencies treebank that the user downloads. Apache OpenNLP bundles no treebank data, distributes none, and ships no models trained on it; the treebanks are used to reproduce accuracy numbers on your own machine. + +## Getting a treebank + +Every UD treebank lives in its own repository under `github.com/UniversalDependencies`, with its splits named `-ud-train.conllu`, `-dev`, and `-test`. The helper next to this file clones one shallowly and lays the splits out under the names the evaluation expects: + +``` +./download-ud-treebank.sh UD_English-EWT /tmp/ud-ewt +``` + +produces `/tmp/ud-ewt/train.conllu` and `/tmp/ud-ewt/test.conllu`. Any treebank that publishes both splits works the same way. + +## Running the gated evaluation + +`ConlluDependencyParserEvalTest` is disabled unless the `opennlp.depparse.ud.dir` system property points at a directory containing `train.conllu` and `test.conllu`: + +``` +./mvnw -pl opennlp-core/opennlp-formats test \ + -Dtest=ConlluDependencyParserEvalTest \ + -Dopennlp.depparse.ud.dir=/tmp/ud-ewt +``` + +Without the property the test reports as skipped, which is why a plain build never needs network access or external data. + +Two properties of the parser worth knowing when reading the numbers: multiword-token sentences are kept because the CoNLL-U reader recovers their dependency rows, and non-projective training sentences are skipped, since the arc-standard transition system cannot derive them; the skip count is inherent to the algorithm, not data loss in the reader. + +## Licensing + +Each treebank carries its own license, stated in its repository README, and downloading one means accepting those terms yourself. The annotations of `UD_English-EWT`, for example, are licensed under CC BY-SA 4.0. The project's handling: treebanks are benchmark inputs on the user's machine only; no treebank data enters the source tree or any release artifact, and the project publishes no models trained on share-alike data. If you train and distribute your own model from a treebank, checking that treebank's terms is your responsibility. diff --git a/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh new file mode 100755 index 0000000000..f229b57a31 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fetches one Universal Dependencies treebank and lays its splits out the way the +# gated dependency-parser evaluation expects: /train.conllu and +# /test.conllu. See README-ud-treebanks.md in this directory for the +# evaluation command and the licensing notes; each treebank carries its own license, +# which you accept by downloading it. Nothing is bundled with Apache OpenNLP. + +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 + echo "" >&2 + echo " treebank-repository a repository name under github.com/UniversalDependencies," >&2 + echo " for example UD_English-EWT" >&2 + echo " target-dir where train.conllu and test.conllu are placed" >&2 + exit 2 +} + +[ $# -ne 2 ] && usage +treebank="$1" +target="$2" + +# Clone shallowly into a temporary directory; only the .conllu files are kept. +clone="$(mktemp -d)" +trap 'rm -rf "${clone}"' EXIT +echo "cloning ${treebank}" +git clone --quiet --depth 1 "https://github.com/UniversalDependencies/${treebank}.git" \ + "${clone}/${treebank}" + +mkdir -p "${target}" +for split in train test; do + # UD names its files -ud-.conllu; the code prefix varies per + # treebank, so match on the stable -ud- suffix. + found="" + for f in "${clone}/${treebank}/"*"-ud-${split}.conllu"; do + [ -e "$f" ] && found="$f" && break + done + if [ -z "${found}" ]; then + echo "no *-ud-${split}.conllu in ${treebank}; the treebank may not publish" >&2 + echo "that split (some hide test data or ship dev only)" >&2 + exit 1 + fi + cp "${found}" "${target}/${split}.conllu" + echo "wrote ${target}/${split}.conllu ($(wc -l < "${target}/${split}.conllu") lines)" +done + +echo "" +echo "run the gated evaluation with:" +echo " ./mvnw -pl opennlp-core/opennlp-formats test \\" +echo " -Dtest=ConlluDependencyParserEvalTest \\" +echo " -Dopennlp.depparse.ud.dir=${target}" From d1072b78e2418f7642bce47f36b811bf8b1c41ff Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 22:06:55 -0400 Subject: [PATCH 43/92] depparse: Refine into a new model, fail loud on unknown transitions, and read blanks and case through the project seams --- .../conllu/ConlluDependencySampleStream.java | 26 +++++++- .../ConlluDependencySampleStreamTest.java | 25 ++++++++ .../depparse/FeedforwardDependencyModel.java | 57 ++++++++++++++--- .../FeedforwardDependencyTrainer.java | 37 ++++++++--- .../FeedforwardDependencyParserTest.java | 61 +++++++++++++++++++ 5 files changed, 188 insertions(+), 18 deletions(-) diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 22f9de9018..9b5154691b 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -31,6 +31,7 @@ import opennlp.tools.depparse.DependencySample; import opennlp.tools.util.InputStreamFactory; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; /** * Reads {@link DependencySample samples} directly from CoNLL-U content, mapping the @@ -112,7 +113,7 @@ private List nextSentence() throws IOException { final List words = new ArrayList<>(); String line; while ((line = reader.readLine()) != null) { - if (line.isBlank()) { + if (isBlank(line)) { if (!words.isEmpty()) { return words; } @@ -133,6 +134,29 @@ private List nextSentence() throws IOException { return words; } + /** + * Determines whether a line separates two sentences, that is whether it is empty or + * consists entirely of whitespace. + * + *

Blankness is decided with {@link StringUtil#isWhitespace(int)} rather than + * {@link String#isBlank()}, because OpenNLP counts the Unicode {@code Zs} category as + * whitespace while the JDK predicate does not: a separator line carrying a stray + * no-break space is still a separator, not a malformed word line.

+ * + * @param line The line to inspect. Must not be {@code null}. + * @return {@code true} if the line is blank, {@code false} otherwise. + */ + private static boolean isBlank(String line) { + for (int i = 0; i < line.length(); ) { + final int codePoint = line.codePointAt(i); + if (!StringUtil.isWhitespace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + /** * Converts one sentence, or returns {@code null} when its annotation is unusable. */ diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index f0a1a40740..01dbfaafb9 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -158,6 +158,31 @@ void testSemanticallyInvalidAnnotationIsSkippedNotFatal() throws IOException { } } + @Test + void testSeparatorLineOfNonBreakingSpaceSeparatesSentences() throws IOException { + // A separator line carrying a stray no-break space is still a separator: OpenNLP + // counts U+00A0 as whitespace, so such a line must not reach the word-line parser + // and abort the stream. + final String content = String.join("\n", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "\u00A0", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"Dogs", "bark"}, first.getTokens()); + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"Fine"}, second.getTokens()); + assertNull(samples.read()); + } + } + @Test void testEmptyContentYieldsNoSample() throws IOException { final InputStreamFactory in = () -> new ByteArrayInputStream(new byte[0]); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index e1457efcab..d19662373f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -27,9 +27,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; -import java.util.Locale; import java.util.Map; +import opennlp.tools.util.StringUtil; + /** * The weights of the feedforward transition parser: embeddings for words, tags, and arc * labels, one hidden layer with cube activation, and a transition output layer, stored @@ -38,8 +39,14 @@ *

This is the pure-Java neural tier: the network is executed with ordinary array * arithmetic, so parsing needs no native runtime, and the same class scores * configurations for training and decoding. Unknown words fall back to a learned - * unknown symbol; words are matched case-insensitively. Instances are immutable and - * safe to share between threads.

+ * unknown symbol; words are matched case-insensitively after + * {@link #normalize(String) normalization}.

+ * + *

An instance is immutable and safe to share between threads once it has been handed + * to a caller. {@link FeedforwardDependencyTrainer} fills the weights while building a + * model and before that model escapes, and + * {@link FeedforwardDependencyTrainer#refine refine} trains a copy rather than the model + * it is given, so no model a caller holds ever changes underneath it.

* * @see FeedforwardDependencyParser * @see FeedforwardDependencyTrainer @@ -68,9 +75,9 @@ public class FeedforwardDependencyModel { Map labelIds, String[] transitions, int embeddingSize, float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias, float[][] outputWeights, float[] outputBias) { - this.wordIds = wordIds; - this.tagIds = tagIds; - this.labelIds = labelIds; + this.wordIds = Map.copyOf(wordIds); + this.tagIds = Map.copyOf(tagIds); + this.labelIds = Map.copyOf(labelIds); this.transitions = transitions; this.embeddingSize = embeddingSize; this.embeddings = embeddings; @@ -142,12 +149,22 @@ public String[] transitions() { return transitions.clone(); } - /** Lowercases a word symbol; special symbols and absences pass through. */ + /** + * Lowercases a word symbol; special symbols and absences pass through. + * + *

Case is mapped with {@link StringUtil#toLowerCase(CharSequence)}, which maps each + * code point through UnicodeData, so no word grows a character on the way into the + * vocabulary and every OpenNLP component derives the same key for the same word.

+ * + * @param word The word to normalize. May be {@code null}. + * @return The vocabulary key of {@code word}, or {@code null} if {@code word} is + * {@code null}. + */ static String normalize(String word) { if (word == null) { return null; } - return word.startsWith("*") ? word : word.toLowerCase(Locale.ROOT); + return word.startsWith("*") ? word : StringUtil.toLowerCase(word); } private static int lookup(Map ids, String symbol) { @@ -289,6 +306,30 @@ private static float[] readVector(DataInputStream data) throws IOException { return vector; } + /** + * Creates an independent copy of this model: the weights are deep-copied, and the + * vocabularies and the transition inventory are shared because they are immutable. + * + *

This lets a training pass update the copy without ever writing to a model a + * caller already holds, which is what keeps the immutability this class documents + * true.

+ * + * @return A copy of this model sharing no mutable state with it. Never {@code null}. + */ + FeedforwardDependencyModel copy() { + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions.clone(), + embeddingSize, copyOf(embeddings), copyOf(hiddenWeights), hiddenBias.clone(), + copyOf(outputWeights), outputBias.clone()); + } + + private static float[][] copyOf(float[][] matrix) { + final float[][] copy = new float[matrix.length][]; + for (int r = 0; r < matrix.length; r++) { + copy[r] = matrix[r].clone(); + } + return copy; + } + Map wordIds() { return wordIds; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index deab7611d6..d1ec3e840b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -169,19 +169,29 @@ public static FeedforwardDependencyModel train(ObjectStream sa * beamed parser scores them, summed log-probabilities, so training optimizes the * quantity decoding uses. * - *

The model is updated in place with per-sentence AdaGrad steps and no dropout; - * {@link Settings#epochs()} counts the refinement passes. Refinement is deterministic - * for a fixed {@link Settings#seed()}. Parse afterwards with the same beam size.

+ *

The refined weights are a copy: {@code model} itself is never written to, so a + * model already being parsed with, possibly by several threads, keeps behaving exactly + * as before while a refined successor is trained from it. The copy is updated with + * per-sentence AdaGrad steps and no dropout; {@link Settings#epochs()} counts the + * refinement passes. Refinement is deterministic for a fixed {@link Settings#seed()}. + * Parse afterwards with the same beam size.

* - * @param model The locally trained model to refine. Must not be {@code null}. + *

The transition inventory comes from {@code model} and is not extended, because + * its size is the width of the trained output layer. A refinement corpus using a + * relation label the original training set lacked is therefore rejected rather than + * silently ignored.

+ * + * @param model The locally trained model to refine. Left untouched. Must not be + * {@code null}. * @param samples The training samples. Must not be {@code null}. * @param settings The hyperparameters; {@code epochs}, {@code learningRate}, * {@code l2}, and {@code seed} apply. Must not be {@code null}. * @param beamSize The beam width to track the gold derivation in. Must be at least 2. - * @return The same model instance, refined. Never {@code null}. + * @return A new refined model, distinct from {@code model}. Never {@code null}. * @throws IOException Thrown if reading the samples fails. * @throws IllegalArgumentException Thrown if a parameter is {@code null}, - * {@code beamSize} is below 2, or no trainable sample can be derived. + * {@code beamSize} is below 2, no trainable sample can be derived, or a sample + * requires a transition {@code model} does not know. */ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model, ObjectStream samples, Settings settings, int beamSize) @@ -215,7 +225,15 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model } final int[] encoded = new int[oracle.size()]; for (int i = 0; i < encoded.length; i++) { - encoded[i] = transitionIds.get(oracle.get(i).encode()); + final String outcome = oracle.get(i).encode(); + final Integer id = transitionIds.get(outcome); + if (id == null) { + // The outcome space was fixed by the original training set, so a relation + // label it never saw has no output unit to push probability onto. + throw new IllegalArgumentException( + "unknown transition in the refinement samples: " + outcome); + } + encoded[i] = id; } trainable.add(s); oracles.add(encoded); @@ -224,7 +242,8 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model throw new IllegalArgumentException("no trainable samples for refinement"); } - final GlobalOptimizer optimizer = new GlobalOptimizer(model, settings); + final FeedforwardDependencyModel refined = model.copy(); + final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings); final Random random = new Random(settings.seed()); final int[] order = new int[trainable.size()]; for (int i = 0; i < order.length; i++) { @@ -246,7 +265,7 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch, loss / Math.max(updates, 1), updates, System.currentTimeMillis() - epochStart); } - return model; + return refined; } /** One candidate path in the refinement beam: the parent link forms the history. */ diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index a4a8c347d8..87597ed5c0 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -21,14 +21,20 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.StringUtil; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -142,6 +148,61 @@ void testRefinementKeepsToyPerformance() throws IOException { assertEquals(1.0d, evaluator.getLas()); } + @Test + void testRefineWithAnUnknownRelationFailsLoud() throws IOException { + // A refinement corpus may carry a relation label the original training set never + // used; the transition inventory is fixed at training time, so refinement cannot + // score it and must say which transition it does not know. + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final List unseenRelation = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "dislocated", "root"})); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(model, + ObjectStreamUtils.createObjectStream(unseenRelation), settings, 2)); + assertEquals("unknown transition in the refinement samples: LEFT_ARC:dislocated", + e.getMessage()); + } + + @Test + void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = local.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + final double[] before = local.score(features); + + final FeedforwardDependencyTrainer.Settings refineSettings = + new FeedforwardDependencyTrainer.Settings(16, 32, 2, 32, 0.01, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + // refinement produces a distinct model, so a model already shared between threads + // cannot change underneath them + assertNotSame(local, refined); + assertArrayEquals(before, local.score(features)); + // and the returned model really carries the refinement + assertFalse(Arrays.equals(before, refined.score(features))); + } + + @Test + void testNormalizeUsesTheUnicodeDataCaseMapping() { + // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's + // String.toLowerCase would render this word as "i" + COMBINING DOT ABOVE instead. + assertEquals(StringUtil.toLowerCase("\u0130STANBUL"), + FeedforwardDependencyModel.normalize("\u0130STANBUL")); + assertEquals("istanbul", FeedforwardDependencyModel.normalize("\u0130STANBUL")); + // special symbols still pass through untouched + assertEquals(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.normalize(FeedforwardDependencyModel.UNKNOWN)); + assertNull(FeedforwardDependencyModel.normalize(null)); + } + @Test void testRefineValidation() { final FeedforwardDependencyTrainer.Settings settings = From 09e82da86d304763626f0a4f928e7a03bfd61b75 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 00:30:33 -0400 Subject: [PATCH 44/92] depparse: Restore the final-sigma rule in vocabulary normalization and make model bytes reproducible Routing normalization through the plain per-code-point mapping broke case-insensitive vocabulary matching for Greek: treebank-derived keys spell a word-final sigma as U+03C2, and an uppercase surface form mapped per code point ends in the medial U+03C3 instead, missing the vocabulary. Normalization now applies the Final_Sigma condition of the Unicode SpecialCasing file, restricted to a single token, on top of the per-code-point mapping, and returns already-lowercase words unchanged without allocating, which is the common case on the parse hot path. Serialized vocabularies are written in ascending id order because the iteration order of the immutable maps is salted per JVM launch, so serializing the same model now produces the same bytes on every run. Transition and dependency-graph relation labels judge blankness under the project whitespace definition, and the copy javadoc no longer calls the cloned transition array immutable. --- .../tools/depparse/DependencyGraph.java | 20 ++++- .../depparse/FeedforwardDependencyModel.java | 63 ++++++++++++-- .../opennlp/tools/depparse/Transition.java | 20 ++++- .../FeedforwardDependencyParserTest.java | 87 +++++++++++++++++++ 4 files changed, 181 insertions(+), 9 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index e95234dcc8..64921d77f3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -22,6 +22,8 @@ import java.util.Collections; import java.util.List; +import opennlp.tools.util.StringUtil; + /** * An immutable dependency tree over one sentence: for every token, the index of its head * and the label of the relation to that head. @@ -77,7 +79,7 @@ public static DependencyGraph of(int[] heads, String[] relations) { } else if (heads[i] == i) { throw new IllegalArgumentException("token " + i + " must not head itself"); } - if (relations[i] == null || relations[i].isBlank()) { + if (relations[i] == null || blank(relations[i])) { throw new IllegalArgumentException("relation of token " + i + " must not be blank"); } } @@ -87,6 +89,22 @@ public static DependencyGraph of(int[] heads, String[] relations) { return new DependencyGraph(heads.clone(), relations.clone()); } + /** + * Reports whether a relation label is blank under the project whitespace + * definition, which unlike the JDK's includes no-break spaces, so a label spelled + * entirely from them cannot pass as a relation. + */ + private static boolean blank(String value) { + for (int i = 0; i < value.length(); ) { + final int cp = value.codePointAt(i); + if (!StringUtil.isWhitespace(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + /** * @return The number of tokens the graph spans. */ diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index d19662373f..cf2d015aaf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -26,7 +26,9 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import opennlp.tools.util.StringUtil; @@ -56,6 +58,12 @@ public class FeedforwardDependencyModel { private static final String MAGIC = "ONLP-FFDP-1"; + /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point with a contextual lowering. */ + private static final int GREEK_CAPITAL_SIGMA = 0x03A3; + + /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the capital. */ + private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2'; + static final String UNKNOWN = "*UNK*"; static final String ABSENT = "*NULL*"; @@ -152,9 +160,15 @@ public String[] transitions() { /** * Lowercases a word symbol; special symbols and absences pass through. * - *

Case is mapped with {@link StringUtil#toLowerCase(CharSequence)}, which maps each - * code point through UnicodeData, so no word grows a character on the way into the - * vocabulary and every OpenNLP component derives the same key for the same word.

+ *

Case is mapped per code point through UnicodeData, the same mapping as + * {@link StringUtil#toLowerCase(CharSequence)}, with one contextual rule on top: a + * Greek capital sigma preceded by a letter and not followed by one lowercases to + * the final form U+03C2, the Final_Sigma condition of the Unicode SpecialCasing + * file restricted to a single token. Natural lowercase Greek text, and with it + * every vocabulary key derived from a treebank, spells a word-final sigma that + * way, so without the rule an uppercase Greek word would normalize to a spelling + * the vocabulary never contains. A word that is already lowercase, the common + * case at parse time, is returned unchanged without allocating.

* * @param word The word to normalize. May be {@code null}. * @return The vocabulary key of {@code word}, or {@code null} if {@code word} is @@ -164,7 +178,36 @@ static String normalize(String word) { if (word == null) { return null; } - return word.startsWith("*") ? word : StringUtil.toLowerCase(word); + if (word.startsWith("*")) { + return word; + } + int i = 0; + while (i < word.length()) { + final int cp = word.codePointAt(i); + if (Character.toLowerCase(cp) != cp) { + break; + } + i += Character.charCount(cp); + } + if (i == word.length()) { + return word; + } + final StringBuilder lowered = new StringBuilder(word.length()); + lowered.append(word, 0, i); + while (i < word.length()) { + final int cp = word.codePointAt(i); + final int width = Character.charCount(cp); + if (cp == GREEK_CAPITAL_SIGMA && i > 0 + && Character.isLetter(word.codePointBefore(i)) + && (i + width >= word.length() + || !Character.isLetter(word.codePointAt(i + width)))) { + lowered.append(GREEK_SMALL_FINAL_SIGMA); + } else { + lowered.appendCodePoint(Character.toLowerCase(cp)); + } + i += width; + } + return lowered.toString(); } private static int lookup(Map ids, String symbol) { @@ -251,7 +294,12 @@ public static FeedforwardDependencyModel load(Path path) throws IOException { private static void writeVocabulary(DataOutputStream data, Map ids) throws IOException { data.writeInt(ids.size()); - for (final Map.Entry entry : ids.entrySet()) { + // Entries are written in ascending id order: the iteration order of the immutable + // maps is salted per JVM launch, and serializing the same model must produce the + // same bytes on every run. + final List> entries = new ArrayList<>(ids.entrySet()); + entries.sort(Map.Entry.comparingByValue()); + for (final Map.Entry entry : entries) { data.writeUTF(entry.getKey()); data.writeInt(entry.getValue()); } @@ -307,8 +355,9 @@ private static float[] readVector(DataInputStream data) throws IOException { } /** - * Creates an independent copy of this model: the weights are deep-copied, and the - * vocabularies and the transition inventory are shared because they are immutable. + * Creates an independent copy of this model: the weights and the transition + * inventory array are deep-copied, and the vocabularies are shared because their + * maps are immutable. * *

This lets a training pass update the copy without ever writing to a model a * caller already holds, which is what keeps the immutability this class documents diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java index 0f9977e675..2cfa6f9282 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -17,6 +17,8 @@ package opennlp.tools.depparse; +import opennlp.tools.util.StringUtil; + /** * One action of the arc-standard transition system: shift the next buffer token onto the * stack, or attach one of the two topmost stack tokens to the other under a relation label. @@ -62,11 +64,27 @@ public enum Type { if (label != null) { throw new IllegalArgumentException("a shift must not carry a label: " + label); } - } else if (label == null || label.isBlank()) { + } else if (label == null || blank(label)) { throw new IllegalArgumentException("an arc transition needs a relation label"); } } + /** + * Reports whether a label is blank under the project whitespace definition, which + * unlike the JDK's includes no-break spaces, so a label spelled entirely from them + * cannot pass as a relation. + */ + private static boolean blank(String value) { + for (int i = 0; i < value.length(); ) { + final int cp = value.codePointAt(i); + if (!StringUtil.isWhitespace(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + /** * Creates a left-arc transition. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 87597ed5c0..2d239df018 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -35,7 +36,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests the pure-Java neural tier end to end: training on a tiny corpus must let the @@ -190,6 +193,90 @@ void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws IOExceptio assertFalse(Arrays.equals(before, refined.score(features))); } + /** + * Pins the one contextual rule layered over the per-code-point mapping: a Greek + * capital sigma preceded by a letter and not followed by one lowercases to the + * final form U+03C2, the way natural lowercase Greek spells it and the way every + * treebank-derived vocabulary key spells it, so an uppercase Greek word normalizes + * to a key the vocabulary can actually contain. The plain per-code-point mapping + * would produce the medial sigma there and miss the vocabulary. + */ + @Test + void testNormalizeAppliesTheFinalSigmaRule() { + // ODOS, road, all caps: the trailing sigma position lowers to U+03C2 + assertEquals("\u03BF\u03B4\u03BF\u03C2", + FeedforwardDependencyModel.normalize("\u039F\u0394\u039F\u03A3")); + // SOFIA: the word-initial sigma is not final and lowers to the medial U+03C3 + assertEquals("\u03C3\u03BF\u03C6\u03B9\u03B1", + FeedforwardDependencyModel.normalize("\u03A3\u039F\u03A6\u0399\u0391")); + // a lone capital sigma has no preceding letter, so the rule does not fire + assertEquals("\u03C3", FeedforwardDependencyModel.normalize("\u03A3")); + } + + /** + * Pins the allocation-free fast path: a word the mapping leaves unchanged, the + * overwhelming majority of parse-time input, is returned as the same instance + * rather than a fresh copy built on every lookup of the scoring loop. + */ + @Test + void testNormalizeReturnsTheSameInstanceForLowercaseWords() { + final String plain = "barks"; + assertSame(plain, FeedforwardDependencyModel.normalize(plain)); + // lowercase Greek with its native final sigma is already normalized + final String greek = "\u03BF\u03B4\u03BF\u03C2"; + assertSame(greek, FeedforwardDependencyModel.normalize(greek)); + } + + /** + * Pins the refinement contract for the sample kind the unknown-transition check + * never sees: a non-projective gold graph has no arc-standard derivation and is + * skipped before the transition inventory is consulted, so an unknown relation + * riding on it must not trigger the unknown-transition failure, and refinement + * proceeds on the remaining projective samples. + */ + @Test + void testNonProjectiveSampleWithUnknownRelationIsSkippedNotFatal() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + // heads {2, 3, -1, 2}: the arcs from 2 to 0 and from 3 to 1 cross, so the graph + // is non-projective, and "dislocated" is a relation the model was never trained on + final List mixed = List.of( + sample(new String[] {"a", "b", "c", "d"}, new String[] {"DT", "NN", "VBZ", "NN"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "dislocated", "root", "obj"}), + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"})); + + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + model, ObjectStreamUtils.createObjectStream(mixed), settings, 2); + assertNotSame(model, refined); + } + + /** + * Pins the serialized vocabulary order: entries are written in ascending id order, + * not in the iteration order of the underlying immutable maps, which the JDK salts + * per launch, so serializing the same model produces the same bytes on every run. + */ + @Test + void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + try (DataInputStream data = new DataInputStream( + new ByteArrayInputStream(out.toByteArray()))) { + data.readUTF(); + for (int vocab = 0; vocab < 3; vocab++) { + final int size = data.readInt(); + int previous = Integer.MIN_VALUE; + for (int entry = 0; entry < size; entry++) { + data.readUTF(); + final int id = data.readInt(); + assertTrue(id > previous, + "vocabulary " + vocab + " must be written in ascending id order"); + previous = id; + } + } + } + } + @Test void testNormalizeUsesTheUnicodeDataCaseMapping() { // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's From 03a005dbc3535f76253371b581aeafcd80b8c1b7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 06:48:34 -0400 Subject: [PATCH 45/92] depparse: Cache hidden-layer contributions per feature pair on frozen models The scorer re-derived every feature's hidden-layer contribution from its embedding on every configuration, although for a frozen model the contribution of a (template position, embedding row) pair is a fixed vector. Parsers now turn on a bounded lazy cache that computes each pair's vector once on first sight and adds it thereafter, the adaptive form of the precomputation described for this architecture by Chen and Manning (2014): tag and label rows are fully cached within a document or two and word rows follow their frequency. Measured on realistic dimensions (20k words, embedding 50, hidden 400, 77 transitions): 5,462 to 71,954 scored states per second, 13.2x. Training and refinement work on uncached copies, copies never carry a cache, concurrent readers are safe by idempotent fill, and a test pins cached-versus-direct agreement to float rounding with identical winning transitions. --- .../depparse/FeedforwardDependencyModel.java | 121 +++++++++++++++++- .../depparse/FeedforwardDependencyParser.java | 3 + .../FeedforwardDependencyParserTest.java | 36 ++++++ 3 files changed, 153 insertions(+), 7 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index cf2d015aaf..ddd906a880 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -30,6 +30,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; import opennlp.tools.util.StringUtil; @@ -67,6 +69,9 @@ public class FeedforwardDependencyModel { static final String UNKNOWN = "*UNK*"; static final String ABSENT = "*NULL*"; + /** The lazy scoring cache; {@code null} until {@link #enableScoringCache()}. */ + private volatile ContributionCache cache; + private final Map wordIds; private final Map tagIds; private final Map labelIds; @@ -107,16 +112,31 @@ public double[] score(int[] features) { final int hidden = hiddenBias.length; final double[] h = new double[hidden]; for (int j = 0; j < hidden; j++) { - final float[] row = hiddenWeights[j]; - double sum = hiddenBias[j]; - for (int f = 0; f < features.length; f++) { - final float[] embedding = embeddings[features[f]]; + h[j] = hiddenBias[j]; + } + final ContributionCache cache = this.cache; + for (int f = 0; f < features.length; f++) { + final int row = features[f]; + final float[] contribution = cache == null ? null : cache.contribution(this, f, row); + if (contribution != null) { + for (int j = 0; j < hidden; j++) { + h[j] += contribution[j]; + } + } else { + final float[] embedding = embeddings[row]; final int offset = f * embeddingSize; - for (int d = 0; d < embeddingSize; d++) { - sum += row[offset + d] * embedding[d]; + for (int j = 0; j < hidden; j++) { + final float[] weights = hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < embeddingSize; d++) { + sum += weights[offset + d] * embedding[d]; + } + h[j] += sum; } } - h[j] = sum * sum * sum; + } + for (int j = 0; j < hidden; j++) { + h[j] = h[j] * h[j] * h[j]; } final double[] scores = new double[transitions.length]; for (int o = 0; o < scores.length; o++) { @@ -130,6 +150,93 @@ public double[] score(int[] features) { return scores; } + /** + * Turns on the scoring cache: the hidden-layer contribution of a (template + * position, embedding row) pair is a fixed vector for a frozen model, so it is + * computed once on first sight and afterwards added instead of being re-derived + * from the embedding on every configuration. Tag and label rows, whose inventories + * are small, are fully cached within a document or two; word rows follow their + * frequency, which is the adaptive form of the precomputation described for this + * architecture by Chen and Manning (2014). + * + *

Cached contributions are rounded to floats once, so scores may differ from the + * uncached path in the last bits; transition decisions are unaffected at any + * realistic margin. The cache is bounded, safe for concurrent readers, and only + * valid on a model whose weights no longer change: training and refinement work on + * uncached copies, and {@link #copy()} never carries a cache over.

+ */ + void enableScoringCache() { + if (cache == null) { + cache = new ContributionCache(2 * FeedforwardContext.POSITIONS + + FeedforwardContext.LABEL_POSITIONS, embeddings.length); + } + } + + /** + * The bounded lazy contribution cache behind {@link #enableScoringCache()}: one + * slot per (template position, embedding row) pair, filled on first use. Filling is + * idempotent, so concurrent readers may compute a contribution twice but never see + * a partial one, and a shared budget bounds the total memory; pairs beyond the + * budget simply keep the direct path. + */ + private static final class ContributionCache { + + /** The most (position, row) pairs the cache will hold. At a hidden size of 400 + * this bounds the cache near 100 MB; typical models stay far below the cap + * because tag and label inventories are small and word usage is Zipf-shaped. */ + private static final int MAX_PAIRS = 65536; + + private final AtomicReferenceArray[] byPosition; + private final AtomicInteger remaining = new AtomicInteger(MAX_PAIRS); + + @SuppressWarnings("unchecked") + private ContributionCache(int positions, int rows) { + byPosition = new AtomicReferenceArray[positions]; + for (int f = 0; f < positions; f++) { + byPosition[f] = new AtomicReferenceArray<>(rows); + } + } + + /** + * Returns the cached hidden-layer contribution of one pair, computing and + * publishing it on first sight while the budget lasts. + * + * @param model The frozen model the contributions derive from. + * @param position The template position. + * @param row The embedding row at that position. + * @return The contribution vector, or {@code null} when the budget is spent and + * the pair is not cached. + */ + private float[] contribution(FeedforwardDependencyModel model, int position, int row) { + final AtomicReferenceArray slots = byPosition[position]; + float[] contribution = slots.get(row); + if (contribution != null) { + return contribution; + } + if (remaining.get() <= 0) { + return null; + } + final int hidden = model.hiddenBias.length; + final float[] embedding = model.embeddings[row]; + final int offset = position * model.embeddingSize; + contribution = new float[hidden]; + for (int j = 0; j < hidden; j++) { + final float[] weights = model.hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < model.embeddingSize; d++) { + sum += weights[offset + d] * embedding[d]; + } + contribution[j] = (float) sum; + } + if (slots.compareAndSet(row, null, contribution)) { + remaining.decrementAndGet(); + } else { + contribution = slots.get(row); + } + return contribution; + } + } + /** * Maps the symbolic features of {@link FeedforwardContext} onto embedding rows. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index aa5b64f622..e058d67d1d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -78,6 +78,9 @@ public FeedforwardDependencyParser(FeedforwardDependencyModel model, int beamSiz } this.model = model; this.beamSize = beamSize; + // A parser only ever reads a frozen model, so the scoring cache is safe to turn + // on here; training and refinement work on uncached copies. + model.enableScoringCache(); final String[] outcomes = model.transitions(); this.transitions = new Transition[outcomes.length]; for (int i = 0; i < outcomes.length; i++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 2d239df018..f431960c09 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -277,6 +277,42 @@ void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws IOException } } + /** + * Pins the scoring cache against the direct path: the parser built in setup turned + * the cache on for the shared model, so scoring the same configuration through a + * fresh uncached copy must agree to float rounding, and the winning transition must + * be identical. Repeated scoring exercises the cache-hit path as well as the + * first-sight fill. + */ + @Test + void testScoringCacheMatchesTheDirectPath() { + final FeedforwardDependencyModel uncached = model.copy(); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = model.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + + for (int round = 0; round < 3; round++) { + final double[] cached = model.score(features); + final double[] direct = uncached.score(features); + assertEquals(direct.length, cached.length); + int bestCached = 0; + int bestDirect = 0; + for (int o = 0; o < cached.length; o++) { + assertEquals(direct[o], cached[o], + Math.max(1.0e-6, Math.abs(direct[o]) * 1.0e-6), + "score " + o + " must agree to float rounding"); + if (cached[o] > cached[bestCached]) { + bestCached = o; + } + if (direct[o] > direct[bestDirect]) { + bestDirect = o; + } + } + assertEquals(bestDirect, bestCached, "the winning transition must be identical"); + } + } + @Test void testNormalizeUsesTheUnicodeDataCaseMapping() { // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's From 299b16f5c9c959267df4b164eac939eee27c44a2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:15:34 -0400 Subject: [PATCH 46/92] depparse: Validate relation labels through StringUtil.isBlank Replaces the two private blank helpers with the shared predicate; behavior is identical since both already followed the toolkit whitespace definition. --- .../opennlp/tools/depparse/DependencyGraph.java | 17 +---------------- .../java/opennlp/tools/depparse/Transition.java | 17 +---------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index 64921d77f3..5c51d20c24 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -79,7 +79,7 @@ public static DependencyGraph of(int[] heads, String[] relations) { } else if (heads[i] == i) { throw new IllegalArgumentException("token " + i + " must not head itself"); } - if (relations[i] == null || blank(relations[i])) { + if (relations[i] == null || StringUtil.isBlank(relations[i])) { throw new IllegalArgumentException("relation of token " + i + " must not be blank"); } } @@ -89,21 +89,6 @@ public static DependencyGraph of(int[] heads, String[] relations) { return new DependencyGraph(heads.clone(), relations.clone()); } - /** - * Reports whether a relation label is blank under the project whitespace - * definition, which unlike the JDK's includes no-break spaces, so a label spelled - * entirely from them cannot pass as a relation. - */ - private static boolean blank(String value) { - for (int i = 0; i < value.length(); ) { - final int cp = value.codePointAt(i); - if (!StringUtil.isWhitespace(cp)) { - return false; - } - i += Character.charCount(cp); - } - return true; - } /** * @return The number of tokens the graph spans. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java index 2cfa6f9282..d72ea26eb8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -64,26 +64,11 @@ public enum Type { if (label != null) { throw new IllegalArgumentException("a shift must not carry a label: " + label); } - } else if (label == null || blank(label)) { + } else if (label == null || StringUtil.isBlank(label)) { throw new IllegalArgumentException("an arc transition needs a relation label"); } } - /** - * Reports whether a label is blank under the project whitespace definition, which - * unlike the JDK's includes no-break spaces, so a label spelled entirely from them - * cannot pass as a relation. - */ - private static boolean blank(String value) { - for (int i = 0; i < value.length(); ) { - final int cp = value.codePointAt(i); - if (!StringUtil.isWhitespace(cp)) { - return false; - } - i += Character.charCount(cp); - } - return true; - } /** * Creates a left-arc transition. From 0e13326f04e98e83249871d7212d52ee5dde27c8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:44:47 -0400 Subject: [PATCH 47/92] dependency: Add a dependency parser manual chapter with a mirror-tested example Add docbkx/dependency.xml, wire it into the manual, and cite ConlluDependencyParserUsageTest. --- opennlp-docs/src/docbkx/dependency.xml | 57 ++++++++++++++++++++++++++ opennlp-docs/src/docbkx/opennlp.xml | 1 + 2 files changed, 58 insertions(+) create mode 100644 opennlp-docs/src/docbkx/dependency.xml diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml new file mode 100644 index 0000000000..a09c1afa54 --- /dev/null +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -0,0 +1,57 @@ + + + + + + + Dependency Parsing + +
+ Introduction + + Dependency parsing assigns each token a syntactic head and a relation label. + DependencyParserME trains on DependencySample streams + and returns a DependencyGraph of heads and relations. CoNLL-U + treebanks are read through ConlluDependencySampleStream. + ConlluDependencyParserUsageTest asserts the train-parse-evaluate + workflow shown here. + +
+ +
+ Training and parsing + + Open a CoNLL-U sample stream, train a model, and parse tokens with their + part-of-speech tags: + + samples = + new ConlluDependencySampleStream(conlluInput, ConlluTagset.U); + +TrainingParameters parameters = TrainingParameters.defaultParams(); +parameters.put(Parameters.CUTOFF_PARAM, 0); +DependencyModel model = DependencyParserME.train("eng", samples, parameters); +DependencyParserME parser = new DependencyParserME(model); + +DependencyGraph graph = parser.parse( + new String[] {"the", "dog", "barks"}, + new String[] {"DET", "NOUN", "VERB"}); +// graph.headOf(0) == 1, graph.relationOf(0) == "det" +// graph.root() is the index of the root token]]> + + A trained model serializes like any other tool model. Evaluation accumulates + unlabeled and labeled attachment scores through + DependencyEvaluator. + +
+ diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 0761fc95ff..4493b16d7d 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -113,6 +113,7 @@ under the License. + From ef853a5bb2ab01be315cce8659fc6be2e7907a9f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:05:44 -0400 Subject: [PATCH 48/92] depparse: Address review: decode outcomes once, fold duplication, name constants - DependencyParserME decodes the model outcome inventory once in the constructor and keeps it as a Transition[]. Decoding a sentence now indexes that array instead of parsing an outcome string per configuration and per outcome, and a model trained for another task is rejected with IllegalArgumentException when the parser is built rather than surfacing as an IllegalStateException in the middle of a sentence. Both constructors document the new failure, and a pinning test builds a parser over a MaxentModel whose outcomes are POS tags to prove the rejection happens up front. - Removed the private isBlank copy from ConlluDependencySampleStream and routed sentence separation through StringUtil.isBlank, so the toolkit whitespace definition lives in one place. The rationale the copy carried moved into the nextSentence javadoc, which also gained its missing @return and @throws. DependencyArc validates its relation through the same predicate, so an arc label made only of a no-break space is rejected exactly as a CoNLL-U separator line is. - StringUtil.isBlank rejects a null argument with IllegalArgumentException instead of letting a NullPointerException escape the loop, and documents it. Its test became a parameterized case list plus an explicit null case. - Extracted the magic values of DependencyContextGenerator into named constants: the word/tag and position separators, the feature count the list is sized to, the valency and distance bounds, and the long-distance feature value. The distance feature is computed once rather than twice. - Named the *ROOT* vocabulary key ROOT_SYMBOL and the "*" special-symbol prefix SPECIAL_SYMBOL_PREFIX on FeedforwardDependencyModel and used them from FeedforwardContext and FeedforwardDependencyTrainer, which spelled all three as literals. FeedforwardContext names the first dependent position instead of indexing the template at a bare 6. - Folded the three repeated special-symbol loops in the trainer vocabulary builder into addSpecialSymbols, and hoisted the repeated model.transitions() call out of the transition-decoding loop. - FeedforwardDependencyModel.score and featureIds validate their array argument, and lookup fails loudly when a vocabulary carries no *UNK* row to fall back on instead of returning null and unboxing to a NullPointerException later. - Documented the package-private accessors of FeedforwardDependencyModel, saying which of them hand out the live arrays the trainer writes into. - Trimmed commentary to what the code does: enableScoringCache drops the literature reference and the restatement of how caching pays off, and DependencyEvaluator.processSample uses {@inheritDoc} plus only what the override adds over the Evaluator contract. - Replaced fully qualified java.util.Arrays, java.util.function.Function, java.io.ByteArrayInputStream, java.io.ByteArrayOutputStream and MaxentModel uses with imports in the trainer, the arc-standard state and the tests, and dropped stray blank lines in DependencyGraph and Transition. - Moved the sample() and corpus() helpers, copied verbatim in three test classes, into a shared DependencyTestSamples fixture with the repetition count as a named constant. - Added pinning tests: a relation of U+00A0 alone is rejected while a label such as nmod:poss is kept, an empty DependencySample is rejected, and the arc and graph relation accessors return what was passed in. --- .../opennlp/tools/depparse/DependencyArc.java | 4 +- .../tools/depparse/DependencyGraph.java | 1 - .../tools/depparse/DependencyGraphTest.java | 9 ++ .../tools/depparse/DependencySampleTest.java | 6 ++ .../conllu/ConlluDependencySampleStream.java | 37 +++----- .../tools/depparse/ArcStandardState.java | 5 +- .../depparse/DependencyContextGenerator.java | 74 ++++++++++------ .../tools/depparse/DependencyEvaluator.java | 6 +- .../tools/depparse/DependencyParserME.java | 42 ++++++--- .../tools/depparse/FeedforwardContext.java | 13 ++- .../depparse/FeedforwardDependencyModel.java | 70 ++++++++++++--- .../depparse/FeedforwardDependencyParser.java | 2 +- .../FeedforwardDependencyTrainer.java | 70 +++++++++------ .../opennlp/tools/depparse/Transition.java | 1 - .../DependencyParserEdgeCaseTest.java | 37 +------- .../depparse/DependencyParserMETest.java | 87 +++++++++++++------ .../tools/depparse/DependencyTestSamples.java | 69 +++++++++++++++ .../FeedforwardDependencyParserTest.java | 23 +---- 18 files changed, 362 insertions(+), 194 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java index bf9dc2eea4..1ab520f5fb 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java @@ -17,6 +17,8 @@ package opennlp.tools.depparse; +import opennlp.tools.util.StringUtil; + /** * One labeled edge of a {@link DependencyGraph}: the token at {@code dependent} is governed * by the token at {@code head} under the given {@code relation}. @@ -56,7 +58,7 @@ public record DependencyArc(int head, int dependent, String relation) { if (head == dependent) { throw new IllegalArgumentException("arc must not be a self-loop: " + head); } - if (relation == null || relation.isBlank()) { + if (relation == null || StringUtil.isBlank(relation)) { throw new IllegalArgumentException("relation must not be null or blank"); } } diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index 5c51d20c24..a694c18f3d 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -89,7 +89,6 @@ public static DependencyGraph of(int[] heads, String[] relations) { return new DependencyGraph(heads.clone(), relations.clone()); } - /** * @return The number of tokens the graph spans. */ diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java index 91bf86bd19..91deee1f98 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -120,6 +120,13 @@ void testOutOfRangeHeadThrows() { void testBlankRelationThrows() { assertThrows(IllegalArgumentException.class, () -> DependencyGraph.of(new int[] {1, -1}, new String[] {" ", "root"})); + // blankness follows the toolkit whitespace definition, which covers the no-break + // space U+00A0 that the JDK predicate leaves out + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {"\u00A0", "root"})); + // and a label that only looks unusual is still content + assertEquals("nmod:poss", DependencyGraph.of(new int[] {1, -1}, + new String[] {"nmod:poss", "root"}).relationOf(0)); } @Test @@ -135,6 +142,8 @@ void testArcValidation() { assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, -1, "det")); assertThrows(IllegalArgumentException.class, () -> new DependencyArc(-2, 0, "det")); assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, " ")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, "\u00A0")); assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, null)); + assertEquals("det", new DependencyArc(1, 0, "det").relation()); } } diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java index 03234eab9f..768b1df4e3 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -68,6 +68,12 @@ void testLengthMismatchThrows() { () -> new DependencySample(TOKENS, new String[] {"DT"}, graph())); } + @Test + void testEmptySampleThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[0], new String[0], graph())); + } + @Test void testInputArraysAreCopied() { final String[] tokens = TOKENS.clone(); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 9b5154691b..782bf5fafc 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -108,12 +108,20 @@ public DependencySample read() throws IOException { /** * Reads the syntactic word lines of the next sentence: comments, multiword token * ranges, and empty nodes are dropped; an empty list means the end of the content. + * + *

Sentences are separated by any line {@link StringUtil#isBlank(CharSequence)} + * accepts, so a separator carrying a stray no-break space still separates rather than + * reaching the word line parser.

+ * + * @return The word lines of the next sentence, or an empty list at the end of the + * content. Never {@code null}. + * @throws IOException Thrown if reading fails or a word line has too few columns. */ private List nextSentence() throws IOException { final List words = new ArrayList<>(); String line; while ((line = reader.readLine()) != null) { - if (isBlank(line)) { + if (StringUtil.isBlank(line)) { if (!words.isEmpty()) { return words; } @@ -135,30 +143,11 @@ private List nextSentence() throws IOException { } /** - * Determines whether a line separates two sentences, that is whether it is empty or - * consists entirely of whitespace. - * - *

Blankness is decided with {@link StringUtil#isWhitespace(int)} rather than - * {@link String#isBlank()}, because OpenNLP counts the Unicode {@code Zs} category as - * whitespace while the JDK predicate does not: a separator line carrying a stray - * no-break space is still a separator, not a malformed word line.

+ * Converts one sentence into a sample. * - * @param line The line to inspect. Must not be {@code null}. - * @return {@code true} if the line is blank, {@code false} otherwise. - */ - private static boolean isBlank(String line) { - for (int i = 0; i < line.length(); ) { - final int codePoint = line.codePointAt(i); - if (!StringUtil.isWhitespace(codePoint)) { - return false; - } - i += Character.charCount(codePoint); - } - return true; - } - - /** - * Converts one sentence, or returns {@code null} when its annotation is unusable. + * @param words The word lines of the sentence. + * @return The converted sample, or {@code null} when the sentence's annotation is + * unusable, for example an underscore head or a graph that is not a tree. */ private DependencySample convert(List words) { final int n = words.size(); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index 8f79eeff1f..c3f3e7147e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -17,6 +17,7 @@ package opennlp.tools.depparse; +import java.util.Arrays; /** * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining @@ -72,8 +73,8 @@ public ArcStandardState(int tokenCount) { this.assignedDependents = new int[tokenCount]; this.leftmostDependents = new int[tokenCount]; this.rightmostDependents = new int[tokenCount]; - java.util.Arrays.fill(this.leftmostDependents, NONE); - java.util.Arrays.fill(this.rightmostDependents, NONE); + Arrays.fill(this.leftmostDependents, NONE); + Arrays.fill(this.rightmostDependents, NONE); } private ArcStandardState(ArcStandardState source) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java index 4639bfcbb5..714c1840db 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -35,6 +35,24 @@ public class DependencyContextGenerator { private static final String ROOT_VALUE = "*ROOT*"; private static final String NONE_VALUE = "*NULL*"; + /** Separates a word from the tag of the same position within one feature. */ + private static final char WORD_TAG_SEPARATOR = '/'; + + /** Separates the parts of a feature combining several positions. */ + private static final char POSITION_SEPARATOR = '|'; + + /** The number of features {@link #getContext(ArcStandardState, String[], String[])} emits. */ + private static final int FEATURE_COUNT = 37; + + /** Valency counts at or above this bound share one feature value. */ + private static final int MAX_VALENCY = 3; + + /** Distances at or above this bound share the {@link #LONG_DISTANCE} feature value. */ + private static final int MAX_DISTANCE = 4; + + /** The feature value standing for every distance of {@link #MAX_DISTANCE} or more. */ + private static final String LONG_DISTANCE = "4+"; + /** * Generates the features of the current configuration. * @@ -75,7 +93,7 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag final String s0rcl = dependentRelation(state, s0, false); final String s1rcl = dependentRelation(state, s1, false); - final List features = new ArrayList<>(36); + final List features = new ArrayList<>(FEATURE_COUNT); features.add("s0w=" + s0w); features.add("s0t=" + s0t); features.add("s1w=" + s1w); @@ -86,20 +104,20 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag features.add("b1w=" + b1w); features.add("b1t=" + b1t); features.add("b2t=" + b2t); - features.add("s0wt=" + s0w + '/' + s0t); - features.add("s1wt=" + s1w + '/' + s1t); - features.add("b0wt=" + b0w + '/' + b0t); - features.add("s0w,b0w=" + s0w + '|' + b0w); - features.add("s0t,b0t=" + s0t + '|' + b0t); - features.add("s0w,b0t=" + s0w + '|' + b0t); - features.add("s0t,b0w=" + s0t + '|' + b0w); - features.add("s0wt,b0t=" + s0w + '/' + s0t + '|' + b0t); - features.add("s1t,s0t=" + s1t + '|' + s0t); - features.add("s1t,s0w=" + s1t + '|' + s0w); - features.add("s1w,s0t=" + s1w + '|' + s0t); - features.add("s1t,s0t,b0t=" + s1t + '|' + s0t + '|' + b0t); - features.add("s0t,b0t,b1t=" + s0t + '|' + b0t + '|' + b1t); - features.add("s2t,s1t,s0t=" + s2t + '|' + s1t + '|' + s0t); + features.add("s0wt=" + s0w + WORD_TAG_SEPARATOR + s0t); + features.add("s1wt=" + s1w + WORD_TAG_SEPARATOR + s1t); + features.add("b0wt=" + b0w + WORD_TAG_SEPARATOR + b0t); + features.add("s0w,b0w=" + s0w + POSITION_SEPARATOR + b0w); + features.add("s0t,b0t=" + s0t + POSITION_SEPARATOR + b0t); + features.add("s0w,b0t=" + s0w + POSITION_SEPARATOR + b0t); + features.add("s0t,b0w=" + s0t + POSITION_SEPARATOR + b0w); + features.add("s0wt,b0t=" + s0w + WORD_TAG_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s1t,s0t=" + s1t + POSITION_SEPARATOR + s0t); + features.add("s1t,s0w=" + s1t + POSITION_SEPARATOR + s0w); + features.add("s1w,s0t=" + s1w + POSITION_SEPARATOR + s0t); + features.add("s1t,s0t,b0t=" + s1t + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s0t,b0t,b1t=" + s0t + POSITION_SEPARATOR + b0t + POSITION_SEPARATOR + b1t); + features.add("s2t,s1t,s0t=" + s2t + POSITION_SEPARATOR + s1t + POSITION_SEPARATOR + s0t); features.add("s0lct=" + s0lct); features.add("s0rct=" + s0rct); features.add("s1lct=" + s1lct); @@ -107,23 +125,24 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag features.add("s0lcl=" + s0lcl); features.add("s0rcl=" + s0rcl); features.add("s1rcl=" + s1rcl); - features.add("s1t,s1rct,s0t=" + s1t + '|' + s1rct + '|' + s0t); - features.add("s0t,s0lct,b0t=" + s0t + '|' + s0lct + '|' + b0t); + features.add("s1t,s1rct,s0t=" + s1t + POSITION_SEPARATOR + s1rct + POSITION_SEPARATOR + s0t); + features.add("s0t,s0lct,b0t=" + s0t + POSITION_SEPARATOR + s0lct + POSITION_SEPARATOR + b0t); features.add("s0deps=" + dependents(state, s0)); features.add("s1deps=" + dependents(state, s1)); - features.add("dist=" + distance(s0, b0)); - features.add("dist,s0t,b0t=" + distance(s0, b0) + '|' + s0t + '|' + b0t); + final String distance = distance(s0, b0); + features.add("dist=" + distance); + features.add("dist,s0t,b0t=" + distance + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); return features.toArray(new String[0]); } - private static String word(String[] tokens, int index) { + private String word(String[] tokens, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; } return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; } - private static String tag(String[] tags, int index) { + private String tag(String[] tags, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; } @@ -131,7 +150,7 @@ private static String tag(String[] tags, int index) { } /** The tag of a token's leftmost or rightmost dependent attached so far. */ - private static String dependentTag(ArcStandardState state, String[] tags, int index, + private String dependentTag(ArcStandardState state, String[] tags, int index, boolean leftmost) { if (index < 0) { return NONE_VALUE; @@ -142,7 +161,7 @@ private static String dependentTag(ArcStandardState state, String[] tags, int in } /** The relation of a token's leftmost or rightmost dependent attached so far. */ - private static String dependentRelation(ArcStandardState state, int index, + private String dependentRelation(ArcStandardState state, int index, boolean leftmost) { if (index < 0) { return NONE_VALUE; @@ -156,15 +175,16 @@ private static String dependentRelation(ArcStandardState state, int index, return relation == null ? NONE_VALUE : relation; } - private static String dependents(ArcStandardState state, int index) { - return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), 3)); + private String dependents(ArcStandardState state, int index) { + return index < 0 ? NONE_VALUE + : Integer.toString(Math.min(state.assignedDependents(index), MAX_VALENCY)); } - private static String distance(int s0, int b0) { + private String distance(int s0, int b0) { if (s0 < 0 || b0 < 0) { return NONE_VALUE; } final int distance = b0 - s0; - return distance >= 4 ? "4+" : Integer.toString(distance); + return distance >= MAX_DISTANCE ? LONG_DISTANCE : Integer.toString(distance); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java index f29bada722..c71c538c5f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java @@ -48,10 +48,10 @@ public DependencyEvaluator(DependencyParser parser) { } /** - * Parses the sample's sentence and scores the prediction against the gold graph. + * {@inheritDoc} * - * @param reference The gold sample. Must not be {@code null}. - * @return A {@link DependencySample} carrying the predicted graph. Never {@code null}. + *

The returned sample carries the predicted graph over the reference tokens, and + * every token of the reference contributes to both scores.

*/ @Override protected DependencySample processSample(DependencySample reference) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java index bbd2d219ff..c565e61724 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -44,12 +44,14 @@ public class DependencyParserME implements DependencyParser { private final MaxentModel model; private final DependencyContextGenerator contextGenerator; + private final Transition[] transitions; /** * Initializes a {@link DependencyParserME} from a {@link DependencyModel}. * * @param model The model to parse with. Must not be {@code null}. - * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. */ public DependencyParserME(DependencyModel model) { if (model == null) { @@ -57,13 +59,15 @@ public DependencyParserME(DependencyModel model) { } this.model = model.getParserModel(); this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(this.model); } /** * Initializes a {@link DependencyParserME} with a raw transition model. * * @param model The transition classification model. Must not be {@code null}. - * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. */ public DependencyParserME(MaxentModel model) { if (model == null) { @@ -71,6 +75,29 @@ public DependencyParserME(MaxentModel model) { } this.model = model; this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(model); + } + + /** + * Decodes the outcome inventory once, so that decoding a sentence indexes it instead + * of parsing an outcome string per configuration and outcome. + * + * @param model The transition classification model. + * @return The transitions by outcome index. Never {@code null}. + * @throws IllegalArgumentException Thrown if an outcome does not decode to a + * transition, which means the model is not a dependency parser model. + */ + private static Transition[] decodeOutcomes(MaxentModel model) { + final Transition[] decoded = new Transition[model.getNumOutcomes()]; + for (int i = 0; i < decoded.length; i++) { + final String outcome = model.getOutcome(i); + try { + decoded[i] = Transition.decode(outcome); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("model outcome is not a transition: " + outcome, e); + } + } + return decoded; } @Override @@ -104,15 +131,8 @@ private Transition bestApplicable(ArcStandardState state, String[] tokens, Strin if (probabilities[i] <= bestProbability) { continue; } - final Transition candidate; - try { - candidate = Transition.decode(model.getOutcome(i)); - } catch (IllegalArgumentException e) { - throw new IllegalStateException( - "model outcome is not a transition: " + model.getOutcome(i), e); - } - if (state.canApply(candidate)) { - best = candidate; + if (state.canApply(transitions[i])) { + best = transitions[i]; bestProbability = probabilities[i]; } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java index 8778071c77..22dec7bf72 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -36,6 +36,9 @@ final class FeedforwardContext { /** The number of dependent positions whose arc label is embedded. */ static final int LABEL_POSITIONS = 8; + /** The index of the first dependent position; the stack and buffer items precede it. */ + private static final int FIRST_DEPENDENT_POSITION = 6; + private FeedforwardContext() { // This class only exposes the static feature template and is never instantiated. } @@ -44,6 +47,12 @@ private FeedforwardContext() { * Extracts the symbolic features of a configuration: {@link #POSITIONS} words, then * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent positions * yield {@code null} entries, which the vocabulary maps to its padding symbol. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The symbolic features, in the order described above. Never {@code null}. */ static String[] extract(ArcStandardState state, String[] tokens, String[] tags) { final int s0 = state.stack(0); @@ -62,7 +71,7 @@ static String[] extract(ArcStandardState state, String[] tokens, String[] tags) features[POSITIONS + i] = symbol(tags, positions[i]); } for (int i = 0; i < LABEL_POSITIONS; i++) { - final int position = positions[6 + i]; + final int position = positions[FIRST_DEPENDENT_POSITION + i]; features[2 * POSITIONS + i] = position >= 0 ? state.assignedRelation(position) : null; } @@ -79,7 +88,7 @@ private static int rightmost(ArcStandardState state, int index) { private static String symbol(String[] values, int index) { if (index == ArcStandardState.ROOT) { - return "*ROOT*"; + return FeedforwardDependencyModel.ROOT_SYMBOL; } return index == ArcStandardState.NONE ? null : values[index]; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index ddd906a880..b881858d50 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -66,9 +66,18 @@ public class FeedforwardDependencyModel { /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the capital. */ private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2'; + /** The vocabulary key of every word, tag, or label the model has no embedding row for. */ static final String UNKNOWN = "*UNK*"; + + /** The vocabulary key of a template position that does not exist in a configuration. */ static final String ABSENT = "*NULL*"; + /** The vocabulary key of the artificial root node. */ + static final String ROOT_SYMBOL = "*ROOT*"; + + /** The prefix marking a vocabulary key as one of the special symbols above. */ + static final String SPECIAL_SYMBOL_PREFIX = "*"; + /** The lazy scoring cache; {@code null} until {@link #enableScoringCache()}. */ private volatile ContributionCache cache; @@ -107,8 +116,12 @@ public class FeedforwardDependencyModel { * {@link #featureIds(String[])}. Must not be {@code null}. * @return One unnormalized score per transition, indexed like * {@link #transitions()}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code features} is {@code null}. */ public double[] score(int[] features) { + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } final int hidden = hiddenBias.length; final double[] h = new double[hidden]; for (int j = 0; j < hidden; j++) { @@ -151,19 +164,15 @@ public double[] score(int[] features) { } /** - * Turns on the scoring cache: the hidden-layer contribution of a (template - * position, embedding row) pair is a fixed vector for a frozen model, so it is - * computed once on first sight and afterwards added instead of being re-derived - * from the embedding on every configuration. Tag and label rows, whose inventories - * are small, are fully cached within a document or two; word rows follow their - * frequency, which is the adaptive form of the precomputation described for this - * architecture by Chen and Manning (2014). + * Turns on the scoring cache: the hidden-layer contribution of a (template position, + * embedding row) pair is a fixed vector while the weights do not change, so it is + * computed once on first sight and afterwards added instead of being re-derived from + * the embedding on every configuration. * *

Cached contributions are rounded to floats once, so scores may differ from the - * uncached path in the last bits; transition decisions are unaffected at any - * realistic margin. The cache is bounded, safe for concurrent readers, and only - * valid on a model whose weights no longer change: training and refinement work on - * uncached copies, and {@link #copy()} never carries a cache over.

+ * uncached path in the last bits. The cache is bounded, safe for concurrent readers, + * and only valid on a model whose weights no longer change: training and refinement + * work on uncached copies, and {@link #copy()} never carries a cache over.

*/ void enableScoringCache() { if (cache == null) { @@ -242,8 +251,12 @@ private float[] contribution(FeedforwardDependencyModel model, int position, int * * @param symbols The symbolic features. Must not be {@code null}. * @return The embedding row per feature. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code symbols} is {@code null}. */ public int[] featureIds(String[] symbols) { + if (symbols == null) { + throw new IllegalArgumentException("symbols must not be null"); + } final int[] ids = new int[symbols.length]; for (int i = 0; i < FeedforwardContext.POSITIONS; i++) { ids[i] = lookup(wordIds, normalize(symbols[i])); @@ -285,7 +298,7 @@ static String normalize(String word) { if (word == null) { return null; } - if (word.startsWith("*")) { + if (word.startsWith(SPECIAL_SYMBOL_PREFIX)) { return word; } int i = 0; @@ -322,6 +335,9 @@ private static int lookup(Map ids, String symbol) { if (id == null) { id = ids.get(UNKNOWN); } + if (id == null) { + throw new IllegalStateException("vocabulary has no " + UNKNOWN + " row to fall back on"); + } return id; } @@ -486,38 +502,68 @@ private static float[][] copyOf(float[][] matrix) { return copy; } + /** + * @return The immutable map from a normalized word to its embedding row. Never {@code null}. + */ Map wordIds() { return wordIds; } + /** + * @return The immutable map from a tag to its embedding row. Never {@code null}. + */ Map tagIds() { return tagIds; } + /** + * @return The immutable map from an arc label to its embedding row. Never {@code null}. + */ Map labelIds() { return labelIds; } + /** + * @return The width of one embedding row. + */ int embeddingSize() { return embeddingSize; } + /** + * @return The live embedding matrix, one row per vocabulary entry, not a copy: the + * trainer writes its updates into it. Never {@code null}. + */ float[][] embeddings() { return embeddings; } + /** + * @return The live hidden layer weights, not a copy. Never {@code null}. + */ float[][] hiddenWeights() { return hiddenWeights; } + /** + * @return The live hidden layer bias, not a copy. Never {@code null}. + */ float[] hiddenBias() { return hiddenBias; } + /** + * @return The live output layer weights, one row per transition, not a copy. Never + * {@code null}. + */ float[][] outputWeights() { return outputWeights; } + /** + * @return The live output layer bias, one entry per transition, not a copy. Never + * {@code null}. + */ float[] outputBias() { return outputBias; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index e058d67d1d..9cd96699cf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -200,7 +200,7 @@ private DependencyGraph beamParse(String[] tokens, String[] tags) { * @param scores The raw output scores. * @return The log-softmax of {@code scores}. Never {@code null}. */ - private static double[] logSoftmax(double[] scores) { + private double[] logSoftmax(double[] scores) { double max = Double.NEGATIVE_INFINITY; for (final double score : scores) { max = Math.max(max, score); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index d1ec3e840b..38aab67d9c 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -19,10 +19,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -137,7 +139,7 @@ public static FeedforwardDependencyModel train(ObjectStream sa * has the wrong dimensionality. */ public static FeedforwardDependencyModel train(ObjectStream samples, - Settings settings, java.util.function.Function pretrained) + Settings settings, Function pretrained) throws IOException { if (samples == null || settings == null) { throw new IllegalArgumentException("samples and settings must not be null"); @@ -207,11 +209,12 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model while ((sample = samples.read()) != null) { corpus.add(sample); } + final String[] outcomes = model.transitions(); final Map transitionIds = new HashMap<>(); - final Transition[] transitions = new Transition[model.transitions().length]; + final Transition[] transitions = new Transition[outcomes.length]; for (int i = 0; i < transitions.length; i++) { - transitionIds.put(model.transitions()[i], i); - transitions[i] = Transition.decode(model.transitions()[i]); + transitionIds.put(outcomes[i], i); + transitions[i] = Transition.decode(outcomes[i]); } final List trainable = new ArrayList<>(); @@ -428,9 +431,9 @@ private double updateFromCandidates(List candidates) { final double logNormalizer = max + Math.log(normalizer); zero(hiddenGradient); - java.util.Arrays.fill(hiddenBiasGradient, 0.0); + Arrays.fill(hiddenBiasGradient, 0.0); zero(outputGradient); - java.util.Arrays.fill(outputBiasGradient, 0.0); + Arrays.fill(outputBiasGradient, 0.0); embeddingGradients.clear(); for (final BeamNode candidate : candidates) { final double weight = Math.exp(candidate.score - logNormalizer) @@ -514,8 +517,8 @@ private void backward(int[] features, int chosen, double weight) { probabilities[o] = Math.exp(probabilities[o] - max); normalizer += probabilities[o]; } - java.util.Arrays.fill(hiddenDelta, 0.0); - java.util.Arrays.fill(inputDelta, 0.0); + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); final float[][] outputWeights = model.outputWeights(); for (int o = 0; o < outputSize; o++) { // dL/dlogit for a path's step under the conditional likelihood: the path weight @@ -552,7 +555,7 @@ private void backward(int[] features, int chosen, double weight) { } /** Turns raw scores into log-probabilities in place. */ - private static void logSoftmaxInPlace(double[] scores) { + private void logSoftmaxInPlace(double[] scores) { double max = Double.NEGATIVE_INFINITY; for (final double score : scores) { max = Math.max(max, score); @@ -570,10 +573,10 @@ private static void logSoftmaxInPlace(double[] scores) { /** Overwrites the random word rows with pretrained vectors where available. */ private static void seed(FeedforwardDependencyModel model, - java.util.function.Function pretrained, Settings settings) { + Function pretrained, Settings settings) { int seeded = 0; for (final Map.Entry entry : model.wordIds().entrySet()) { - if (entry.getKey().startsWith("*")) { + if (entry.getKey().startsWith(FeedforwardDependencyModel.SPECIAL_SYMBOL_PREFIX)) { // The special unknown, padding, and root symbols have no pretrained // counterpart, so they keep their random initialization. continue; @@ -623,28 +626,22 @@ private static FeedforwardDependencyModel initialize(List corp int row = 0; final Map wordIds = new HashMap<>(); - for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, - FeedforwardDependencyModel.ABSENT, "*ROOT*")) { - wordIds.put(special, row++); - } + row = addSpecialSymbols(wordIds, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); for (final Map.Entry entry : wordCounts.entrySet()) { if (entry.getValue() >= settings.wordCutoff() && !wordIds.containsKey(entry.getKey())) { wordIds.put(entry.getKey(), row++); } } final Map tags = new HashMap<>(); - for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, - FeedforwardDependencyModel.ABSENT, "*ROOT*")) { - tags.put(special, row++); - } + row = addSpecialSymbols(tags, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); for (final String tag : tagIds.keySet()) { tags.put(tag, row++); } final Map labels = new HashMap<>(); - for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, - FeedforwardDependencyModel.ABSENT)) { - labels.put(special, row++); - } + row = addSpecialSymbols(labels, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT); for (final String label : labelIds.keySet()) { labels.put(label, row++); } @@ -671,6 +668,23 @@ private static FeedforwardDependencyModel initialize(List corp new float[settings.hiddenSize()], outputWeights, new float[transitions.length]); } + /** + * Assigns the next embedding rows to the special symbols of one vocabulary. + * + * @param ids The vocabulary to fill. + * @param row The next free embedding row. + * @param symbols The special symbols, in the order they take their rows. + * @return The next free embedding row after the symbols. + */ + private static int addSpecialSymbols(Map ids, int row, String... symbols) { + int next = row; + for (final String symbol : symbols) { + ids.put(symbol, next); + next++; + } + return next; + } + /** Replays the oracle over every projective sample, emitting one example per step. */ private static void collectExamples(List corpus, FeedforwardDependencyModel model, List featureList, List goldList) { @@ -760,9 +774,9 @@ private static void optimize(FeedforwardDependencyModel model, List featu final int batchEnd = Math.min(batchStart + settings.batchSize(), exampleCount); final int batch = batchEnd - batchStart; zero(hiddenGradient); - java.util.Arrays.fill(hiddenBiasGradient, 0.0); + Arrays.fill(hiddenBiasGradient, 0.0); zero(outputGradient); - java.util.Arrays.fill(outputBiasGradient, 0.0); + Arrays.fill(outputBiasGradient, 0.0); embeddingGradients.clear(); for (int b = batchStart; b < batchEnd; b++) { @@ -810,8 +824,8 @@ private static void optimize(FeedforwardDependencyModel model, List featu } loss -= Math.log(Math.max(probabilities[goldTransition], 1e-12)); - java.util.Arrays.fill(hiddenDelta, 0.0); - java.util.Arrays.fill(inputDelta, 0.0); + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); for (int o = 0; o < outputSize; o++) { final double delta = probabilities[o] - (o == goldTransition ? 1.0 : 0.0); outputBiasGradient[o] += delta; @@ -903,7 +917,7 @@ private static float[][] uniform(Random random, int rows, int columns, double sc private static void zero(double[][] matrix) { for (final double[] row : matrix) { - java.util.Arrays.fill(row, 0.0); + Arrays.fill(row, 0.0); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java index d72ea26eb8..8095e48102 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -69,7 +69,6 @@ public enum Type { } } - /** * Creates a left-arc transition. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java index f7015089e9..4d46e04022 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java @@ -32,6 +32,8 @@ import opennlp.tools.util.Parameters; import opennlp.tools.util.TrainingParameters; +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -48,41 +50,6 @@ public class DependencyParserEdgeCaseTest { private static FeedforwardDependencyModel feedforwardModel; private static FeedforwardDependencyParser feedforwardParser; - /** - * Builds one gold sample from its parallel arrays. - * - * @param tokens The sentence tokens. Must not be {@code null}. - * @param tags The part-of-speech tags aligned with {@code tokens}. - * @param heads The zero-based head per token, {@code -1} for the root. - * @param relations The relation label per token. - * @return The assembled sample. Never {@code null}. - */ - private static DependencySample sample(String[] tokens, String[] tags, int[] heads, - String[] relations) { - return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); - } - - /** - * Builds the projective training corpus: three tiny sentences, each repeated so both - * trainers see enough evidence to memorize them. - * - * @return The training samples. Never {@code null}. - */ - private static List corpus() { - final List distinct = List.of( - sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, - new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), - sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, - new int[] {1, -1}, new String[] {"nsubj", "root"}), - sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, - new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); - final List corpus = new ArrayList<>(); - for (int i = 0; i < 40; i++) { - corpus.addAll(distinct); - } - return corpus; - } - /** * Builds a four-token sample whose gold arcs (2,0) and (3,1) cross, so the tree is * non-projective and has no arc-standard derivation. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java index 62550ad239..716506435c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -17,17 +17,19 @@ package opennlp.tools.depparse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import opennlp.tools.ml.model.MaxentModel; import opennlp.tools.util.ObjectStreamUtils; import opennlp.tools.util.Parameters; import opennlp.tools.util.TrainingParameters; +import static opennlp.tools.depparse.DependencyTestSamples.corpus; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -41,26 +43,6 @@ public class DependencyParserMETest { private static DependencyModel model; private static DependencyParserME parser; - private static DependencySample sample(String[] tokens, String[] tags, int[] heads, - String[] relations) { - return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); - } - - private static List corpus() { - final List distinct = List.of( - sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, - new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), - sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, - new int[] {1, -1}, new String[] {"nsubj", "root"}), - sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, - new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); - final List corpus = new ArrayList<>(); - for (int i = 0; i < 40; i++) { - corpus.addAll(distinct); - } - return corpus; - } - @BeforeAll static void trainParser() throws IOException { final TrainingParameters parameters = TrainingParameters.defaultParams(); @@ -113,7 +95,7 @@ void testConstructorRejectsNullModel() { assertThrows(IllegalArgumentException.class, () -> new DependencyParserME((DependencyModel) null)); assertThrows(IllegalArgumentException.class, - () -> new DependencyParserME((opennlp.tools.ml.model.MaxentModel) null)); + () -> new DependencyParserME((MaxentModel) null)); } @Test @@ -131,10 +113,10 @@ void testTrainValidatesArguments() { @Test void testModelRoundTripThroughSerialization() throws IOException { - final java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); model.serialize(out); final DependencyModel reloaded = new DependencyModel( - new java.io.ByteArrayInputStream(out.toByteArray())); + new ByteArrayInputStream(out.toByteArray())); final DependencyGraph parsed = new DependencyParserME(reloaded) .parse(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}); assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, @@ -146,4 +128,59 @@ void testModelRejectsNullParserModel() { assertThrows(IllegalArgumentException.class, () -> new DependencyModel("eng", null, null)); } + + @Test + void testModelWithForeignOutcomesIsRejectedAtConstruction() { + // The outcome inventory is decoded once up front, so a model trained for another + // task is rejected when the parser is built rather than mid-sentence. + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME(new OutcomeOnlyModel("NN", "VB"))); + } + + /** + * A {@link MaxentModel} that only knows its outcome inventory, enough to build a + * parser from; any other use fails. + */ + private record OutcomeOnlyModel(String... outcomes) implements MaxentModel { + + @Override + public String getOutcome(int i) { + return outcomes[i]; + } + + @Override + public int getNumOutcomes() { + return outcomes.length; + } + + @Override + public double[] eval(String[] context) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, double[] probs) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, float[] values) { + throw new UnsupportedOperationException(); + } + + @Override + public String getBestOutcome(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public String getAllOutcomes(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public int getIndex(String outcome) { + throw new UnsupportedOperationException(); + } + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java new file mode 100644 index 0000000000..051668a585 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * The gold samples shared by the dependency parser tests. + */ +final class DependencyTestSamples { + + /** How often the distinct sentences are repeated in {@link #corpus()}. */ + private static final int REPETITIONS = 40; + + private DependencyTestSamples() { + // This class only exposes static sample builders and is never instantiated. + } + + /** + * Builds one gold sample from its parallel arrays. + * + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @param heads The zero-based head per token, {@code -1} for the root. + * @param relations The relation label per token. + * @return The assembled sample. Never {@code null}. + */ + static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + /** + * Builds the projective training corpus: three tiny sentences, each repeated so both + * trainers see enough evidence to memorize them. + * + * @return The training samples. Never {@code null}. + */ + static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(REPETITIONS * distinct.size()); + for (int i = 0; i < REPETITIONS; i++) { + corpus.addAll(distinct); + } + return corpus; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index f431960c09..d08426ee38 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -21,7 +21,6 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,6 +30,8 @@ import opennlp.tools.util.ObjectStreamUtils; import opennlp.tools.util.StringUtil; +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -50,26 +51,6 @@ public class FeedforwardDependencyParserTest { private static FeedforwardDependencyModel model; private static FeedforwardDependencyParser parser; - private static DependencySample sample(String[] tokens, String[] tags, int[] heads, - String[] relations) { - return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); - } - - private static List corpus() { - final List distinct = List.of( - sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, - new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), - sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, - new int[] {1, -1}, new String[] {"nsubj", "root"}), - sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, - new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); - final List corpus = new ArrayList<>(); - for (int i = 0; i < 40; i++) { - corpus.addAll(distinct); - } - return corpus; - } - @BeforeAll static void trainParser() throws IOException { // dropout off so the tiny network memorizes deterministically From 89825abbcbcdda211575088a9f36be637771ef26 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:53:28 -0400 Subject: [PATCH 49/92] depparse: Address review: cite specs and document private helpers Add reference links for CoNLL-U, the arc-standard system (Nivre 2004), the feedforward architecture and training recipe (Chen and Manning 2014), early update (Collins and Roark 2004), and the Unicode SpecialCasing file. State that thread safety is implementation specific on the DependencyParser interface. Add javadoc to the remaining private helpers in main and test code. Fold the duplicated corpus reading in the feedforward trainer into a readAll helper, and read the corrupt-model test fixture as UTF-8. The DependencyModel serialVersionUID was verified to equal the serialver default. --- .../tools/depparse/DependencyGraph.java | 9 ++++ .../tools/depparse/DependencyParser.java | 2 + .../tools/depparse/DependencyGraphTest.java | 1 + .../tools/depparse/DependencySampleTest.java | 1 + .../conllu/ConlluDependencySampleStream.java | 11 +++- .../ConlluDependencyParserEvalTest.java | 1 + .../ConlluDependencySampleStreamTest.java | 3 ++ .../tools/depparse/ArcStandardState.java | 20 ++++++- .../depparse/DependencyContextGenerator.java | 4 ++ .../tools/depparse/FeedforwardContext.java | 6 ++- .../depparse/FeedforwardDependencyModel.java | 36 ++++++++++++- .../FeedforwardDependencyTrainer.java | 52 +++++++++++++------ .../tools/depparse/ArcStandardOracleTest.java | 6 +++ .../depparse/DependencyParserMETest.java | 6 +++ .../FeedforwardDependencyParserTest.java | 10 +++- 15 files changed, 143 insertions(+), 25 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index a694c18f3d..40a1f9c90b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -39,6 +39,9 @@ public final class DependencyGraph { private final int[] heads; private final String[] relations; + /** + * Wraps already validated arrays; instances are created through {@link #of}. + */ private DependencyGraph(int[] heads, String[] relations) { this.heads = heads; this.relations = relations; @@ -144,6 +147,12 @@ public List arcs() { return Collections.unmodifiableList(arcs); } + /** + * Rejects a token index outside {@code [0, size())}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ private void checkIndex(int index) { if (index < 0 || index >= heads.length) { throw new IllegalArgumentException("token index out of range: " + index diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java index 0c6e5d1303..bb9b4029e3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java @@ -27,6 +27,8 @@ * whose indices refer back to the input token array, so spans computed for those tokens * remain valid for the parse.

* + *

Thread safety is implementation specific.

+ * * @see DependencyGraph * @since 3.0.0 */ diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java index 91deee1f98..3229e339f8 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -30,6 +30,7 @@ */ public class DependencyGraphTest { + /** The three-token graph shared by the accessor tests. */ private static DependencyGraph sample() { return DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java index 768b1df4e3..6be16f4817 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -31,6 +31,7 @@ public class DependencySampleTest { private static final String[] TOKENS = {"the", "dog", "barks"}; private static final String[] TAGS = {"DT", "NN", "VBZ"}; + /** The graph matching {@link #TOKENS} and {@link #TAGS}. */ private static DependencyGraph graph() { return DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 782bf5fafc..e62222768b 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -34,8 +34,9 @@ import opennlp.tools.util.StringUtil; /** - * Reads {@link DependencySample samples} directly from CoNLL-U content, mapping the - * {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * Reads {@link DependencySample samples} directly from + * CoNLL-U content, mapping + * the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. * *

The file is parsed raw, deliberately not through {@link ConlluStream}: that stream * merges multiword token ranges with their syntactic words, which suits the token and @@ -185,6 +186,12 @@ public void close() throws IOException { reader.close(); } + /** + * Opens a fresh UTF-8 reader over the content. + * + * @return A reader positioned at the start of the content. Never {@code null}. + * @throws IOException Thrown if opening the content fails. + */ private BufferedReader open() throws IOException { return new BufferedReader( new InputStreamReader(in.createInputStream(), StandardCharsets.UTF_8)); diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java index af4cc1e919..fd0934ee9f 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -77,6 +77,7 @@ void testTrainAndScoreOnUniversalDependencies() throws IOException { assertTrue(evaluator.getLas() > 0.5d, "LAS regressed below the floor"); } + /** Opens a sample stream over one CoNLL-U split using the universal tagset. */ private static ConlluDependencySampleStream samples(Path conllu) throws IOException { final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); return new ConlluDependencySampleStream(in, ConlluTagset.U); diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index 01dbfaafb9..460e531a7f 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -40,6 +40,7 @@ */ public class ConlluDependencySampleStreamTest { + /** Joins the ten CoNLL-U columns of one word line with tabs. */ private static String line(String... fields) { return String.join("\t", fields); } @@ -68,10 +69,12 @@ private static String line(String... fields) { line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), "") + "\n"; + /** An in-memory factory over the shared fixture. */ private static InputStreamFactory factory() { return () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); } + /** A stream over the shared fixture using the universal tagset. */ private static ConlluDependencySampleStream stream() throws IOException { return new ConlluDependencySampleStream(factory(), ConlluTagset.U); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index c3f3e7147e..afa2646e9a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -21,7 +21,8 @@ /** * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining - * tokens, and the arcs assigned so far. + * tokens, and the arcs assigned so far. The arc-standard transition system is described + * in Nivre (2004). * *

The stack bottom holds the artificial root, exposed as {@link #ROOT}. Positions that * do not exist, such as the second stack element in the initial configuration, are exposed @@ -77,6 +78,9 @@ public ArcStandardState(int tokenCount) { Arrays.fill(this.rightmostDependents, NONE); } + /** + * Deep-copies {@code source}; used only by {@link #copy()}. + */ private ArcStandardState(ArcStandardState source) { this.tokenCount = source.tokenCount; this.stack = source.stack.clone(); @@ -157,6 +161,14 @@ public void apply(Transition transition) { } } + /** + * Records the arc from {@code head} to {@code dependent} and updates the dependent + * bookkeeping of {@code head} when it is a token rather than the artificial root. + * + * @param head The head token index, or {@link #ROOT} for the artificial root. + * @param dependent The zero-based index of the token being attached. + * @param relation The relation label of the arc. + */ private void attach(int head, int dependent, String relation) { heads[dependent] = head; relations[dependent] = relation; @@ -231,6 +243,12 @@ public int assignedDependents(int index) { return assignedDependents[index]; } + /** + * Rejects a token index outside {@code [0, tokenCount)}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ private void checkTokenIndex(int index) { if (index < 0 || index >= tokenCount) { throw new IllegalArgumentException("token index out of range: " + index); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java index 714c1840db..bcd7ab732d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -135,6 +135,7 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag return features.toArray(new String[0]); } + /** The word at a position, or the marker value for the root and absent positions. */ private String word(String[] tokens, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; @@ -142,6 +143,7 @@ private String word(String[] tokens, int index) { return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; } + /** The tag at a position, or the marker value for the root and absent positions. */ private String tag(String[] tags, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; @@ -175,11 +177,13 @@ private String dependentRelation(ArcStandardState state, int index, return relation == null ? NONE_VALUE : relation; } + /** A token's dependent count so far, capped at {@link #MAX_VALENCY}. */ private String dependents(ArcStandardState state, int index) { return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), MAX_VALENCY)); } + /** The bucketed distance between stack top and buffer front, capped at {@link #MAX_DISTANCE}. */ private String distance(int s0, int b0) { if (s0 < 0 || b0 < 0) { return NONE_VALUE; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java index 22dec7bf72..29fb8e4c87 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -20,7 +20,8 @@ /** * The feature template of the feedforward parser: a fixed set of configuration * positions whose words, tags, and arc labels are embedded and concatenated into the - * network input. + * network input, following + * Chen and Manning (2014). * *

Positions: the top three stack and buffer items; the leftmost and rightmost * dependents of the top two stack items; and the leftmost dependent of the leftmost @@ -78,14 +79,17 @@ static String[] extract(ArcStandardState state, String[] tokens, String[] tags) return features; } + /** The leftmost dependent of a position, or the absence marker for absent positions. */ private static int leftmost(ArcStandardState state, int index) { return index >= 0 ? state.leftmostDependent(index) : ArcStandardState.NONE; } + /** The rightmost dependent of a position, or the absence marker for absent positions. */ private static int rightmost(ArcStandardState state, int index) { return index >= 0 ? state.rightmostDependent(index) : ArcStandardState.NONE; } + /** The value at a position: the root symbol for the root, {@code null} when absent. */ private static String symbol(String[] values, int index) { if (index == ArcStandardState.ROOT) { return FeedforwardDependencyModel.ROOT_SYMBOL; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index b881858d50..dca28791b7 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -38,7 +38,9 @@ /** * The weights of the feedforward transition parser: embeddings for words, tags, and arc * labels, one hidden layer with cube activation, and a transition output layer, stored - * in a plain versioned binary format with no serialization framework involved. + * in a plain versioned binary format with no serialization framework involved. The + * architecture follows + * Chen and Manning (2014). * *

This is the pure-Java neural tier: the network is executed with ordinary array * arithmetic, so parsing needs no native runtime, and the same class scores @@ -283,7 +285,8 @@ public String[] transitions() { *

Case is mapped per code point through UnicodeData, the same mapping as * {@link StringUtil#toLowerCase(CharSequence)}, with one contextual rule on top: a * Greek capital sigma preceded by a letter and not followed by one lowercases to - * the final form U+03C2, the Final_Sigma condition of the Unicode SpecialCasing + * the final form U+03C2, the Final_Sigma condition of the Unicode + * SpecialCasing * file restricted to a single token. Natural lowercase Greek text, and with it * every vocabulary key derived from a treebank, spells a word-final sigma that * way, so without the rule an uppercase Greek word would normalize to a spelling @@ -330,6 +333,14 @@ static String normalize(String word) { return lowered.toString(); } + /** + * Resolves a symbol to its embedding row: absences map to {@link #ABSENT}, symbols + * without a row of their own fall back to {@link #UNKNOWN}. + * + * @param ids The vocabulary to resolve against. + * @param symbol The symbol to resolve, or {@code null} for an absent position. + * @return The embedding row of the symbol or of its fallback. + */ private static int lookup(Map ids, String symbol) { Integer id = ids.get(symbol == null ? ABSENT : symbol); if (id == null) { @@ -414,6 +425,9 @@ public static FeedforwardDependencyModel load(Path path) throws IOException { } } + /** + * Writes one vocabulary as its size followed by (symbol, id) pairs. + */ private static void writeVocabulary(DataOutputStream data, Map ids) throws IOException { data.writeInt(ids.size()); @@ -428,6 +442,9 @@ private static void writeVocabulary(DataOutputStream data, Map } } + /** + * Reads one vocabulary written by {@link #writeVocabulary}. + */ private static Map readVocabulary(DataInputStream data) throws IOException { final int size = data.readInt(); @@ -439,6 +456,9 @@ private static Map readVocabulary(DataInputStream data) return ids; } + /** + * Writes a rectangular matrix as its dimensions followed by its values in row order. + */ private static void writeMatrix(DataOutputStream data, float[][] matrix) throws IOException { data.writeInt(matrix.length); @@ -450,6 +470,9 @@ private static void writeMatrix(DataOutputStream data, float[][] matrix) } } + /** + * Reads a matrix written by {@link #writeMatrix}. + */ private static float[][] readMatrix(DataInputStream data) throws IOException { final int rows = data.readInt(); final int columns = data.readInt(); @@ -462,6 +485,9 @@ private static float[][] readMatrix(DataInputStream data) throws IOException { return matrix; } + /** + * Writes a vector as its length followed by its values. + */ private static void writeVector(DataOutputStream data, float[] vector) throws IOException { data.writeInt(vector.length); for (final float value : vector) { @@ -469,6 +495,9 @@ private static void writeVector(DataOutputStream data, float[] vector) throws IO } } + /** + * Reads a vector written by {@link #writeVector}. + */ private static float[] readVector(DataInputStream data) throws IOException { final float[] vector = new float[data.readInt()]; for (int i = 0; i < vector.length; i++) { @@ -494,6 +523,9 @@ embeddingSize, copyOf(embeddings), copyOf(hiddenWeights), hiddenBias.clone(), copyOf(outputWeights), outputBias.clone()); } + /** + * Deep-copies a matrix, row by row. + */ private static float[][] copyOf(float[][] matrix) { final float[][] copy = new float[matrix.length][]; for (int r = 0; r < matrix.length; r++) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index 38aab67d9c..7f7e0c07ed 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -34,9 +34,10 @@ /** * Trains the {@link FeedforwardDependencyModel} entirely in Java: oracle-derived * transition examples, minibatch AdaGrad over a softmax cross-entropy loss, cube - * activation, and inverted dropout on the hidden layer. No external training framework - * is involved, so the whole neural tier, training and inference, is plain array - * arithmetic inside the JVM. + * activation, and inverted dropout on the hidden layer, the training recipe of + * Chen and Manning (2014). No external + * training framework is involved, so the whole neural tier, training and inference, is + * plain array arithmetic inside the JVM. * *

Words below the frequency cutoff share a learned unknown embedding; absent * template positions share a learned padding embedding. Non-projective samples have no @@ -144,11 +145,7 @@ public static FeedforwardDependencyModel train(ObjectStream sa if (samples == null || settings == null) { throw new IllegalArgumentException("samples and settings must not be null"); } - final List corpus = new ArrayList<>(); - DependencySample sample; - while ((sample = samples.read()) != null) { - corpus.add(sample); - } + final List corpus = readAll(samples); final FeedforwardDependencyModel model = initialize(corpus, settings); if (pretrained != null) { seed(model, pretrained, settings); @@ -166,10 +163,11 @@ public static FeedforwardDependencyModel train(ObjectStream sa /** * Fine-tunes a locally trained model globally: sentences are decoded with a beam, the * gold derivation is tracked through it, and the moment the gold prefix falls out of - * the beam an early update pushes the model toward keeping it. The loss is a - * conditional likelihood over the beam's candidate paths, scored exactly like the - * beamed parser scores them, summed log-probabilities, so training optimizes the - * quantity decoding uses. + * the beam an early update, in the sense of + * Collins and Roark (2004), pushes + * the model toward keeping it. The loss is a conditional likelihood over the beam's + * candidate paths, scored exactly like the beamed parser scores them, summed + * log-probabilities, so training optimizes the quantity decoding uses. * *

The refined weights are a copy: {@code model} itself is never written to, so a * model already being parsed with, possibly by several threads, keeps behaving exactly @@ -204,11 +202,7 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model if (beamSize < 2) { throw new IllegalArgumentException("beamSize must be at least 2: " + beamSize); } - final List corpus = new ArrayList<>(); - DependencySample sample; - while ((sample = samples.read()) != null) { - corpus.add(sample); - } + final List corpus = readAll(samples); final String[] outcomes = model.transitions(); final Map transitionIds = new HashMap<>(); final Transition[] transitions = new Transition[outcomes.length]; @@ -280,6 +274,7 @@ private static final class BeamNode { private final boolean gold; private ArcStandardState state; + /** Extends {@code parent} by one transition; the start node passes {@code null}. */ private BeamNode(BeamNode parent, int[] features, int transition, double score, boolean gold) { this.parent = parent; @@ -318,6 +313,7 @@ private static final class GlobalOptimizer { private final double[] hiddenDelta; private final double[] inputDelta; + /** Sizes the accumulators and scratch buffers for one refinement run over {@code model}. */ private GlobalOptimizer(FeedforwardDependencyModel model, Settings settings) { this.model = model; this.settings = settings; @@ -571,6 +567,23 @@ private void logSoftmaxInPlace(double[] scores) { } } + /** + * Reads a sample stream into memory; both trainers pass over the corpus repeatedly. + * + * @param samples The stream to drain. + * @return All samples in stream order. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + */ + private static List readAll(ObjectStream samples) + throws IOException { + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + return corpus; + } + /** Overwrites the random word rows with pretrained vectors where available. */ private static void seed(FeedforwardDependencyModel model, Function pretrained, Settings settings) { @@ -880,6 +893,7 @@ private static void optimize(FeedforwardDependencyModel model, List featu } } + /** One AdaGrad step on a weight matrix, with the L2 penalty folded into the gradient. */ private static void update(float[][] weights, double[][] gradients, double[][] accumulators, int batch, Settings settings) { for (int r = 0; r < weights.length; r++) { @@ -895,6 +909,7 @@ private static void update(float[][] weights, double[][] gradients, } } + /** One AdaGrad step on a bias vector; biases carry no L2 penalty. */ private static void updateVector(float[] weights, double[] gradients, double[] accumulators, int batch, Settings settings) { for (int i = 0; i < weights.length; i++) { @@ -905,6 +920,7 @@ private static void updateVector(float[] weights, double[] gradients, } } + /** A matrix drawn uniformly from {@code [-scale, scale]}. */ private static float[][] uniform(Random random, int rows, int columns, double scale) { final float[][] matrix = new float[rows][columns]; for (int r = 0; r < rows; r++) { @@ -915,12 +931,14 @@ private static float[][] uniform(Random random, int rows, int columns, double sc return matrix; } + /** Fills a matrix with zeros. */ private static void zero(double[][] matrix) { for (final double[] row : matrix) { Arrays.fill(row, 0.0); } } + /** Fisher-Yates shuffle of the visit order. */ private static void shuffle(int[] order, Random random) { for (int i = order.length - 1; i > 0; i--) { final int j = random.nextInt(i + 1); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java index 6cfa9f57ca..4ded778794 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java @@ -31,6 +31,12 @@ */ public class ArcStandardOracleTest { + /** + * Derives the oracle transitions for {@code gold} and replays them on a fresh state. + * + * @param gold The gold graph to derive from. + * @return The graph the replayed derivation builds. Never {@code null}. + */ private static DependencyGraph replay(DependencyGraph gold) { final List transitions = ArcStandardOracle.transitions(gold); // every token is shifted once and attached once diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java index 716506435c..aa839c5fbf 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -43,6 +43,12 @@ public class DependencyParserMETest { private static DependencyModel model; private static DependencyParserME parser; + /** + * Trains the shared model once for all tests; the zero cutoff keeps every feature of + * the tiny corpus. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ @BeforeAll static void trainParser() throws IOException { final TrainingParameters parameters = TrainingParameters.defaultParams(); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index d08426ee38..6d7437587e 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -51,9 +52,14 @@ public class FeedforwardDependencyParserTest { private static FeedforwardDependencyModel model; private static FeedforwardDependencyParser parser; + /** + * Trains the shared model once for all tests, with dropout off and a fixed seed so + * the tiny network memorizes the corpus deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ @BeforeAll static void trainParser() throws IOException { - // dropout off so the tiny network memorizes deterministically final FeedforwardDependencyTrainer.Settings settings = new FeedforwardDependencyTrainer.Settings(16, 32, 120, 32, 0.05, 0.0, 0.0, 1, 17L); model = FeedforwardDependencyTrainer.train( @@ -340,7 +346,7 @@ void testModelRoundTripThroughSerialization() throws IOException { @Test void testCorruptModelFailsLoud() { assertThrows(IOException.class, () -> FeedforwardDependencyModel.load( - new ByteArrayInputStream("not a model".getBytes()))); + new ByteArrayInputStream("not a model".getBytes(StandardCharsets.UTF_8)))); } @Test From e69d298204b1bd5d004e5a3822b748d57d3d9f8d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 21 Aug 2026 20:58:27 -0400 Subject: [PATCH 50/92] OPENNLP-547: Parse CoNLL-U fields without regex --- .../conllu/ConlluDependencySampleStream.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index e62222768b..f1df227783 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -131,7 +131,7 @@ private List nextSentence() throws IOException { if (line.charAt(0) == '#') { continue; } - final String[] fields = line.split("\t", -1); + final String[] fields = splitFields(line); if (fields.length < COLUMNS) { throw new IOException("not a CoNLL-U word line: " + line); } @@ -143,6 +143,25 @@ private List nextSentence() throws IOException { return words; } + /** + * Splits a CoNLL-U word line into its tab-delimited fields, retaining empty fields. + * + * @param line The line to split. + * @return The fields in source order. Never {@code null}. + */ + private String[] splitFields(String line) { + final List fields = new ArrayList<>(); + int fieldStart = 0; + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) == '\t') { + fields.add(line.substring(fieldStart, i)); + fieldStart = i + 1; + } + } + fields.add(line.substring(fieldStart)); + return fields.toArray(String[]::new); + } + /** * Converts one sentence into a sample. * From 0e610d785594f0264abfb9652bdbe2158a1f5847 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 14 Jul 2026 21:29:46 -0400 Subject: [PATCH 51/92] depparse: DependencyAnnotator, the container's first graph-shaped layer Wires the dependency parser into the document pipeline: reads the token and tag layers, parses, and provides a dependencies layer with one DependencyArc per token anchored on the dependent's span. Arc head and dependent are indices into the token layer, exercising the container rule that annotations reference each other by layer and index; the test resolves an arc's head through the token layer back to its span in the original text. (cherry picked from commit 902fbb3f570900d057dcd033ea9ee7c084599911) --- .../tools/depparse/DependencyAnnotator.java | 100 ++++++++++++++++++ .../depparse/DependencyAnnotatorTest.java | 93 ++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java new file mode 100644 index 0000000000..806db6daef --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link DependencyParser} to the document pipeline: reads + * {@link Layers#TOKENS} and {@link Layers#POS_TAGS} and provides + * {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's span. + * + *

This is the first graph-shaped layer: an arc's {@link DependencyArc#head()} and + * {@link DependencyArc#dependent()} are indices into the token layer, following the + * container's rule that annotations reference each other by layer and index, never by + * object identity.

+ * + * @since 3.0.0 + */ +public class DependencyAnnotator implements DocumentAnnotator { + + /** + * Dependency arcs; one annotation per token, aligned with {@link Layers#TOKENS} by + * position, anchored on the dependent token's span. + */ + public static final LayerKey DEPENDENCIES = + LayerKey.of("dependencies", DependencyArc.class); + + private final DependencyParser parser; + + /** + * Initializes the adapter. + * + * @param parser The dependency parser to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyAnnotator(DependencyParser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tokens.isEmpty() || tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final String[] words = new String[tokens.size()]; + final String[] posTags = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + posTags[i] = tags.get(i).value(); + } + final DependencyGraph graph = parser.parse(words, posTags); + final List> arcs = new ArrayList<>(graph.size()); + for (final DependencyArc arc : graph.arcs()) { + arcs.add(new Annotation<>(tokens.get(arc.dependent()).span(), arc)); + } + return document.with(DEPENDENCIES, arcs); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS, Layers.POS_TAGS); + } + + @Override + public Set> provides() { + return Set.of(DEPENDENCIES); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java new file mode 100644 index 0000000000..c047d20c0e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyAnnotator} as the container's first graph-shaped layer: arcs + * reference tokens by layer index, and resolving an arc through the token layer lands on + * the right span of the original text. + */ +public class DependencyAnnotatorTest { + + private static final DependencyParser FIXED = (tokens, tags) -> + DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + + private static Document tokenized() { + return Document.of("the dog barks") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(8, 13), "barks"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "DT"), + new Annotation<>(new Span(4, 7), "NN"), + new Annotation<>(new Span(8, 13), "VBZ"))); + } + + @Test + void testArcsResolveThroughTheTokenLayer() { + final Document document = new DependencyAnnotator(FIXED).annotate(tokenized()); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(3, arcs.size()); + + // the arc of "dog" is anchored on the dependent's span + final Annotation dog = arcs.get(1); + assertEquals(new Span(4, 7), dog.span()); + assertEquals("nsubj", dog.value().relation()); + + // cross-layer reference: the head index resolves into the token layer + final List> tokens = document.get(Layers.TOKENS); + final Annotation head = tokens.get(dog.value().head()); + assertEquals("barks", head.value()); + assertEquals("barks", head.span().getCoveredText(document.text()).toString()); + } + + @Test + void testRootArcCarriesRootHead() { + final Document document = new DependencyAnnotator(FIXED).annotate(tokenized()); + final DependencyArc root = + document.get(DependencyAnnotator.DEPENDENCIES).get(2).value(); + assertEquals(DependencyArc.ROOT_HEAD, root.head()); + assertEquals("root", root.relation()); + } + + @Test + void testMissingLayersThrow() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no layers"))); + } + + @Test + void testNullParserThrows() { + assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(null)); + } +} From c174f7beef1356885c901ecf5dbfb774c040f598 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:50:45 -0400 Subject: [PATCH 52/92] depparse: Pipeline and contract tests for the dependency annotator, document-coordinate javadoc --- .../tools/depparse/DependencyAnnotator.java | 28 ++ .../DependencyAnnotatorEdgeCaseTest.java | 119 ++++++++ .../DependencyAnnotatorPipelineTest.java | 288 ++++++++++++++++++ .../depparse/DependencyAnnotatorTest.java | 16 +- 4 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index 806db6daef..9f02d365e0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -37,6 +37,13 @@ * container's rule that annotations reference each other by layer and index, never by * object identity.

* + *

The whole token layer is handed to the parser as one sequence, so the result is a + * single tree over all tokens of the document and the arc indices are positions in the + * document-wide token layer. Because every token span already refers to the original + * document text, anchoring an arc on its dependent token's span puts the arc in document + * coordinates without any offset arithmetic, no matter which sentence the token came + * from.

+ * * @since 3.0.0 */ public class DependencyAnnotator implements DocumentAnnotator { @@ -63,6 +70,23 @@ public DependencyAnnotator(DependencyParser parser) { this.parser = parser; } + /** + * Parses the document's token layer and adds the {@link #DEPENDENCIES} layer. + * + *

The token and tag values are read in layer order and passed to the parser as one + * sequence. The resulting arcs are emitted in token order, so the new layer is aligned + * with {@link Layers#TOKENS} by position, and each arc annotation reuses the span of + * its dependent token.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry a + * non-empty {@link Layers#TOKENS} layer plus a {@link Layers#POS_TAGS} + * layer of equal size. + * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * token layer is absent or empty, or the tag layer does not have exactly one + * tag per token. + */ @Override public Document annotate(Document document) { if (document == null) { @@ -74,6 +98,7 @@ public Document annotate(Document document) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } + // unwrap the aligned layers into the parallel arrays the parser interface expects final String[] words = new String[tokens.size()]; final String[] posTags = new String[tokens.size()]; for (int i = 0; i < words.length; i++) { @@ -81,6 +106,9 @@ public Document annotate(Document document) { posTags[i] = tags.get(i).value(); } final DependencyGraph graph = parser.parse(words, posTags); + // graph.arcs() is in token order with one arc per token; anchoring each arc on its + // dependent token's span keeps the layer aligned with the token layer and puts the + // arc in document coordinates, since token spans refer to the original text final List> arcs = new ArrayList<>(graph.size()); for (final DependencyArc arc : graph.arcs()) { arcs.add(new Annotation<>(tokens.get(arc.dependent()).span(), arc)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java new file mode 100644 index 0000000000..2e05260a1d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pins down the boundary behavior of {@link DependencyAnnotator}: the exact exception and + * message for empty and misaligned input layers, the immutability rule that a second + * annotation pass is rejected, and the exact {@code requires()} and {@code provides()} + * declarations the pipeline validation relies on. + */ +public class DependencyAnnotatorEdgeCaseTest { + + /** + * A parser stub that returns a fixed two-token graph regardless of its input, so the + * assertions in this class exercise only the annotator's own layer handling. + */ + private static final DependencyParser FIXED = (tokens, tags) -> + DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, + new String[] {"root", "obj"}); + + /** + * Builds a document over the text {@code "ab cd"} carrying aligned two-entry token and + * tag layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * + * @return A document ready for dependency annotation. Never {@code null}. + */ + private static Document twoTokens() { + return Document.of("ab cd") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + } + + @Test + void testEmptyTokenAndTagLayersAreRejected() { + // a document with zero sentences has zero tokens; the annotator refuses to parse it + final Document empty = Document.of("") + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(empty)); + assertEquals("document needs aligned tokens and pos layers", + e.getMessage()); + } + + @Test + void testMisalignedTagLayerIsRejected() { + // two tokens but only one tag: the layers are present yet not aligned by position + final Document misaligned = Document.of("ab cd") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(misaligned)); + assertEquals("document needs aligned tokens and pos layers", + e.getMessage()); + } + + @Test + void testAnnotatingTwiceIsRejected() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + final Document once = annotator.annotate(twoTokens()); + assertEquals(2, once.get(DependencyAnnotator.DEPENDENCIES).size()); + + // documents are immutable and layers are add-once: a second pass must not overwrite + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(once)); + assertEquals("layer is already present: dependencies", e.getMessage()); + } + + @Test + void testRequiresAndProvidesDeclarationsAreExact() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + assertEquals(Set.of(Layers.TOKENS, Layers.POS_TAGS), annotator.requires()); + assertEquals(Set.of(DependencyAnnotator.DEPENDENCIES), annotator.provides()); + } + + @Test + void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { + // requires() feeds the analyzer's validation: no tokenizer or tagger, no pipeline + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new DependencyAnnotator(FIXED)); + assertThrows(IllegalArgumentException.class, builder::build); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java new file mode 100644 index 0000000000..f93f22f15f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.document.POSTaggerAnnotator; +import opennlp.tools.document.SentenceDetectorAnnotator; +import opennlp.tools.document.TokenizerAnnotator; +import opennlp.tools.postag.POSTagger; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Demonstrates {@link DependencyAnnotator} at the end of a complete {@link DocumentAnalyzer} + * pipeline: raw text goes in, and the dependency layer comes out anchored on the original + * text. The upstream steps are deliberately simple inline implementations of the task + * interfaces, and the parser is a {@link DependencyParserME} trained here on a tiny corpus + * it can memorize, so every expected head, relation, and span is exact and reproducible. + * + *

The central property under test is coordinate anchoring for multi-sentence input: the + * annotator hands the whole token layer to the parser as one sequence, so the arcs it gets + * back carry indices into the document-wide token layer, and each arc's annotation must sit + * on its dependent token's span in document coordinates. For the second sentence this only + * works when the token layer itself was anchored correctly, which the assertions verify by + * reading the covered text of the arcs' spans back out of the original document.

+ */ +public class DependencyAnnotatorPipelineTest { + + /** + * The two-sentence input text; sentence one covers offsets 0..14 and sentence two covers + * offsets 15..29 of the original document. + */ + private static final String TEXT = "the dog barks. she eats fish."; + + /** + * Maps every token of the test corpus to its part-of-speech tag, standing in for a + * trained tagger. + */ + private static final Map LEXICON = Map.of( + "the", "DT", "dog", "NN", "barks", "VBZ", + "she", "PRP", "eats", "VBZ", "fish", "NN"); + + /** + * A sentence detector stub that closes a sentence after every period and skips the one + * following blank, producing sentence spans in document coordinates. + */ + private static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the pipeline only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A tokenizer stub that treats blanks and periods as token boundaries, so word tokens + * come out without trailing punctuation. + */ + private static final Tokenizer WORD_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the pipeline only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = + i == s.length() || s.charAt(i) == ' ' || s.charAt(i) == '.'; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A dictionary tagger over {@link #LEXICON} that fails loud on any token the test corpus + * does not define, so a tokenization mistake cannot silently degrade the parse. + */ + private static final POSTagger LEXICON_TAGGER = new POSTagger() { + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + final String tag = LEXICON.get(sentence[i]); + if (tag == null) { + throw new IllegalArgumentException("token is not in the test lexicon: " + sentence[i]); + } + tags[i] = tag; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the pipeline only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the pipeline only calls tag"); + } + }; + + private static DependencyParserME parser; + + /** + * Builds the training corpus: the token sequence of the two-sentence document with its + * gold tree, plus a one-token sentence, each repeated often enough for the model to + * memorize them. The document-wide sequence has a single root at {@code barks} with the + * second predicate attached as {@code parataxis}, because a dependency graph always + * forms one tree over the token sequence it is built for. + * + * @return The training samples. Never {@code null} or empty. + */ + private static List corpus() { + final List distinct = List.of( + new DependencySample( + new String[] {"the", "dog", "barks", "she", "eats", "fish"}, + new String[] {"DT", "NN", "VBZ", "PRP", "VBZ", "NN"}, + DependencyGraph.of(new int[] {1, 2, -1, 4, 2, 4}, + new String[] {"det", "nsubj", "root", "nsubj", "parataxis", "obj"})), + new DependencySample(new String[] {"barks"}, new String[] {"VBZ"}, + DependencyGraph.of(new int[] {-1}, new String[] {"root"}))); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + /** + * Trains the shared parser once for all tests. The trainer is deterministic for a fixed + * corpus and fixed parameters, so the assertions below hold on every run. + * + * @throws IOException Thrown if training fails, which fails the test class. + */ + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + parser = new DependencyParserME(DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters)); + } + + /** + * Assembles the complete pipeline: sentence splitting, tokenization, tagging, and + * dependency parsing with the trained model. + * + * @return A {@link DocumentAnalyzer} ready to analyze raw text. Never {@code null}. + */ + private static DocumentAnalyzer pipeline() { + return DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(WORD_TOKENIZER)) + .add(new POSTaggerAnnotator(LEXICON_TAGGER)) + .add(new DependencyAnnotator(parser)) + .build(); + } + + @Test + void testTwoSentenceTextYieldsOneExactArcPerTokenInDocumentCoordinates() { + final Document document = pipeline().analyze(TEXT); + + // sanity of the upstream layers the dependency annotator consumed + assertEquals(2, document.get(Layers.SENTENCES).size()); + assertEquals(6, document.get(Layers.TOKENS).size()); + + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(6, arcs.size()); + + // the memorized gold tree, with every span in document coordinates + final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, 2, 4}; + final String[] relations = {"det", "nsubj", "root", "nsubj", "parataxis", "obj"}; + final Span[] spans = {new Span(0, 3), new Span(4, 7), new Span(8, 13), + new Span(15, 18), new Span(19, 23), new Span(24, 28)}; + for (int i = 0; i < arcs.size(); i++) { + final Annotation arc = arcs.get(i); + assertEquals(spans[i], arc.span(), "span of arc " + i); + assertEquals(heads[i], arc.value().head(), "head of arc " + i); + assertEquals(i, arc.value().dependent(), "dependent of arc " + i); + assertEquals(relations[i], arc.value().relation(), "relation of arc " + i); + } + } + + @Test + void testSecondSentenceArcsResolveToTheOriginalText() { + final Document document = pipeline().analyze(TEXT); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + + // "she" is token 3 of the document-wide token layer, not token 0 of its sentence + final Annotation she = arcs.get(3); + assertEquals(new Span(15, 18), she.span()); + assertEquals("she", she.span().getCoveredText(document.text()).toString()); + assertEquals("nsubj", she.value().relation()); + + // its head index is likewise document-wide: 4 points at "eats", never 1 at "dog" + assertEquals(4, she.value().head()); + final Annotation head = document.get(Layers.TOKENS).get(she.value().head()); + assertEquals("eats", head.value()); + assertEquals(new Span(19, 23), head.span()); + assertEquals("eats", head.span().getCoveredText(document.text()).toString()); + } + + @Test + void testSingleTokenSentenceParsesToARootArc() { + final Document document = pipeline().analyze("barks."); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(1, arcs.size()); + assertEquals(new Span(0, 5), arcs.get(0).span()); + assertEquals(DependencyArc.ROOT_HEAD, arcs.get(0).value().head()); + assertEquals(0, arcs.get(0).value().dependent()); + assertEquals("root", arcs.get(0).value().relation()); + } + + @Test + void testTextWithZeroSentencesFailsBeforeTheDependencyAnnotatorRuns() { + // empty text yields no sentences and no tokens, so the tagger already fails loud + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> pipeline().analyze("")); + assertEquals("document lacks the required layer tokens", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java index c047d20c0e..e1f6938d3c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -36,9 +36,20 @@ */ public class DependencyAnnotatorTest { + /** + * A parser stub that always returns the gold graph of {@code "the dog barks"}, so the + * assertions in this class depend only on the annotator's own layer handling and not on + * any trained model. + */ private static final DependencyParser FIXED = (tokens, tags) -> DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + /** + * Builds a document over the text {@code "the dog barks"} carrying aligned token and tag + * layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * + * @return A document ready for dependency annotation. Never {@code null}. + */ private static Document tokenized() { return Document.of("the dog barks") .with(Layers.TOKENS, List.of( @@ -58,12 +69,13 @@ void testArcsResolveThroughTheTokenLayer() { document.get(DependencyAnnotator.DEPENDENCIES); assertEquals(3, arcs.size()); - // the arc of "dog" is anchored on the dependent's span + // the arc of "dog" is anchored on the dependent token's span in the original text final Annotation dog = arcs.get(1); assertEquals(new Span(4, 7), dog.span()); assertEquals("nsubj", dog.value().relation()); - // cross-layer reference: the head index resolves into the token layer + // cross-layer reference: the arc stores its head as an index, and looking that index + // up in the token layer lands on the head token and its span in the original text final List> tokens = document.get(Layers.TOKENS); final Annotation head = tokens.get(dog.value().head()); assertEquals("barks", head.value()); From edaa98f768610d45930e66379e5880d4d541d8b7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 02:47:16 -0400 Subject: [PATCH 53/92] depparse: Parse each sentence separately in the dependency annotator --- .../tools/depparse/DependencyAnnotator.java | 96 +++++++++++++------ .../DependencyAnnotatorEdgeCaseTest.java | 74 +++++++++++++- .../DependencyAnnotatorPipelineTest.java | 40 ++++---- .../depparse/DependencyAnnotatorTest.java | 7 +- 4 files changed, 166 insertions(+), 51 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index 9f02d365e0..dcc5aebd35 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -29,20 +29,22 @@ /** * Adapts a {@link DependencyParser} to the document pipeline: reads - * {@link Layers#TOKENS} and {@link Layers#POS_TAGS} and provides - * {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's span. + * {@link Layers#SENTENCES}, {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and + * provides {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's + * span. * *

This is the first graph-shaped layer: an arc's {@link DependencyArc#head()} and * {@link DependencyArc#dependent()} are indices into the token layer, following the * container's rule that annotations reference each other by layer and index, never by * object identity.

* - *

The whole token layer is handed to the parser as one sequence, so the result is a - * single tree over all tokens of the document and the arc indices are positions in the - * document-wide token layer. Because every token span already refers to the original - * document text, anchoring an arc on its dependent token's span puts the arc in document - * coordinates without any offset arithmetic, no matter which sentence the token came - * from.

+ *

Each sentence is parsed separately, the way the parser is trained, so every + * sentence gets its own tree and its own root arc. The sentence-local indices the + * parser produces are shifted by the sentence's first token position, which keeps every + * arc's head and dependent a position in the document-wide token layer. Because every + * token span already refers to the original document text, anchoring an arc on its + * dependent token's span puts the arc in document coordinates without further offset + * arithmetic.

* * @since 3.0.0 */ @@ -71,21 +73,25 @@ public DependencyAnnotator(DependencyParser parser) { } /** - * Parses the document's token layer and adds the {@link #DEPENDENCIES} layer. + * Parses the document sentence by sentence and adds the {@link #DEPENDENCIES} layer. * - *

The token and tag values are read in layer order and passed to the parser as one - * sequence. The resulting arcs are emitted in token order, so the new layer is aligned - * with {@link Layers#TOKENS} by position, and each arc annotation reuses the span of - * its dependent token.

+ *

For every sentence, the tokens whose spans lie inside the sentence span are + * passed to the parser with their tags as one sequence, and the resulting + * sentence-local arcs are shifted by the sentence's first token position. Arcs are + * emitted in token order, so the new layer is aligned with {@link Layers#TOKENS} by + * position, and each arc annotation reuses the span of its dependent token. A + * sentence containing no tokens contributes no arcs.

* * @param document The document to annotate. Must not be {@code null} and must carry a - * non-empty {@link Layers#TOKENS} layer plus a {@link Layers#POS_TAGS} - * layer of equal size. + * non-empty {@link Layers#SENTENCES} layer, a non-empty + * {@link Layers#TOKENS} layer whose every token lies inside a + * sentence, and a {@link Layers#POS_TAGS} layer of equal size. * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * token layer is absent or empty, or the tag layer does not have exactly one - * tag per token. + * token layer is absent or empty, the tag layer does not have exactly one + * tag per token, the sentence layer is absent or empty, or a token lies + * outside every sentence. */ @Override public Document annotate(Document document) { @@ -98,27 +104,55 @@ public Document annotate(Document document) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } - // unwrap the aligned layers into the parallel arrays the parser interface expects - final String[] words = new String[tokens.size()]; - final String[] posTags = new String[tokens.size()]; - for (int i = 0; i < words.length; i++) { - words[i] = tokens.get(i).value(); - posTags[i] = tags.get(i).value(); + final List> sentences = document.get(Layers.SENTENCES); + if (sentences.isEmpty()) { + throw new IllegalArgumentException( + "document needs a non-empty " + Layers.SENTENCES + " layer"); } - final DependencyGraph graph = parser.parse(words, posTags); - // graph.arcs() is in token order with one arc per token; anchoring each arc on its - // dependent token's span keeps the layer aligned with the token layer and puts the - // arc in document coordinates, since token spans refer to the original text - final List> arcs = new ArrayList<>(graph.size()); - for (final DependencyArc arc : graph.arcs()) { - arcs.add(new Annotation<>(tokens.get(arc.dependent()).span(), arc)); + final List> arcs = new ArrayList<>(tokens.size()); + // Walk the token layer once: both layers are in text order, so each sentence + // consumes the contiguous run of tokens whose spans it encloses. + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + // unwrap the sentence's slice into the parallel arrays the parser expects + final String[] words = new String[count]; + final String[] posTags = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + posTags[i] = tags.get(first + i).value(); + } + final DependencyGraph graph = parser.parse(words, posTags); + // The parser indexes within the sentence; shifting by the sentence's first token + // position turns every head and dependent into a document-wide token index, and + // anchoring each arc on its dependent token's span puts the arc in document + // coordinates, since token spans refer to the original text. + for (final DependencyArc arc : graph.arcs()) { + final int head = arc.head() == DependencyArc.ROOT_HEAD + ? DependencyArc.ROOT_HEAD : arc.head() + first; + arcs.add(new Annotation<>(tokens.get(first + arc.dependent()).span(), + new DependencyArc(head, arc.dependent() + first, arc.relation()))); + } + } + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); } return document.with(DEPENDENCIES, arcs); } @Override public Set> requires() { - return Set.of(Layers.TOKENS, Layers.POS_TAGS); + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); } @Override diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index 2e05260a1d..e1b33d2a3d 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -55,6 +55,7 @@ public class DependencyAnnotatorEdgeCaseTest { */ private static Document twoTokens() { return Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "ab cd"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 2), "ab"), new Annotation<>(new Span(3, 5), "cd"))) @@ -105,10 +106,81 @@ void testAnnotatingTwiceIsRejected() { @Test void testRequiresAndProvidesDeclarationsAreExact() { final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); - assertEquals(Set.of(Layers.TOKENS, Layers.POS_TAGS), annotator.requires()); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + annotator.requires()); assertEquals(Set.of(DependencyAnnotator.DEPENDENCIES), annotator.provides()); } + /** + * Verifies the per-sentence contract: two one-token sentences are parsed as two + * separate calls, each yielding its own root arc, and the dependents come back as + * document-wide token indices. + */ + @Test + void testEachSentenceGetsItsOwnTree() { + final DependencyParser oneTokenRoot = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + final Document document = Document.of("ab. cd.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "ab."), + new Annotation<>(new Span(4, 7), "cd."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(4, 6), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(4, 6), "VB"))); + + final List> arcs = + new DependencyAnnotator(oneTokenRoot).annotate(document) + .get(DependencyAnnotator.DEPENDENCIES); + assertEquals(2, arcs.size()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 0, "root"), + arcs.get(0).value()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), + arcs.get(1).value()); + assertEquals(new Span(0, 2), arcs.get(0).span()); + assertEquals(new Span(4, 6), arcs.get(1).span()); + } + + /** + * Verifies the sentence-layer requirements fail loud: a token-bearing document + * without a sentence layer is rejected, and so is a token lying outside every + * sentence. + */ + @Test + void testSentenceLayerProblemsAreRejected() { + final Document noSentences = Document.of("ab cd") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + final IllegalArgumentException missing = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(noSentences)); + assertEquals("document needs a non-empty sentences layer", + missing.getMessage()); + + final Document strayToken = Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 2), "ab"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(strayToken)); + assertEquals("token at [3..5) lies outside every sentence", stray.getMessage()); + } + @Test void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { // requires() feeds the analyzer's validation: no tokenizer or tagger, no pipeline diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index f93f22f15f..97991c2754 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -52,11 +52,12 @@ * it can memorize, so every expected head, relation, and span is exact and reproducible. * *

The central property under test is coordinate anchoring for multi-sentence input: the - * annotator hands the whole token layer to the parser as one sequence, so the arcs it gets - * back carry indices into the document-wide token layer, and each arc's annotation must sit - * on its dependent token's span in document coordinates. For the second sentence this only - * works when the token layer itself was anchored correctly, which the assertions verify by - * reading the covered text of the arcs' spans back out of the original document.

+ * annotator parses each sentence separately and shifts the sentence-local arcs by the + * sentence's first token position, so every head and dependent is an index into the + * document-wide token layer and each arc's annotation sits on its dependent token's span in + * document coordinates. For the second sentence this only works when the shift is applied + * consistently, which the assertions verify by reading the covered text of the arcs' spans + * back out of the original document.

*/ public class DependencyAnnotatorPipelineTest { @@ -167,21 +168,25 @@ public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { private static DependencyParserME parser; /** - * Builds the training corpus: the token sequence of the two-sentence document with its - * gold tree, plus a one-token sentence, each repeated often enough for the model to - * memorize them. The document-wide sequence has a single root at {@code barks} with the - * second predicate attached as {@code parataxis}, because a dependency graph always - * forms one tree over the token sequence it is built for. + * Builds the training corpus: the two sentences of the example document with their + * gold trees, plus a one-token sentence, each repeated often enough for the model to + * memorize them. Every sample is one sentence, the unit the annotator hands to the + * parser, so each sentence carries its own root. * * @return The training samples. Never {@code null} or empty. */ private static List corpus() { final List distinct = List.of( new DependencySample( - new String[] {"the", "dog", "barks", "she", "eats", "fish"}, - new String[] {"DT", "NN", "VBZ", "PRP", "VBZ", "NN"}, - DependencyGraph.of(new int[] {1, 2, -1, 4, 2, 4}, - new String[] {"det", "nsubj", "root", "nsubj", "parataxis", "obj"})), + new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}, + DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"})), + new DependencySample( + new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"}, + DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"})), new DependencySample(new String[] {"barks"}, new String[] {"VBZ"}, DependencyGraph.of(new int[] {-1}, new String[] {"root"}))); final List corpus = new ArrayList<>(); @@ -232,9 +237,10 @@ void testTwoSentenceTextYieldsOneExactArcPerTokenInDocumentCoordinates() { document.get(DependencyAnnotator.DEPENDENCIES); assertEquals(6, arcs.size()); - // the memorized gold tree, with every span in document coordinates - final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, 2, 4}; - final String[] relations = {"det", "nsubj", "root", "nsubj", "parataxis", "obj"}; + // the memorized gold trees, one root per sentence, every span and head index in + // document coordinates + final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, DependencyArc.ROOT_HEAD, 4}; + final String[] relations = {"det", "nsubj", "root", "nsubj", "root", "obj"}; final Span[] spans = {new Span(0, 3), new Span(4, 7), new Span(8, 13), new Span(15, 18), new Span(19, 23), new Span(24, 28)}; for (int i = 0; i < arcs.size(); i++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java index e1f6938d3c..554cb3a72d 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -45,13 +45,16 @@ public class DependencyAnnotatorTest { DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); /** - * Builds a document over the text {@code "the dog barks"} carrying aligned token and tag - * layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * Builds a document over the text {@code "the dog barks"} carrying a one-sentence layer plus + * aligned token and tag layers, mirroring what the upstream sentence, tokenizer, and + * tagger annotators would produce. * * @return A document ready for dependency annotation. Never {@code null}. */ private static Document tokenized() { return Document.of("the dog barks") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 13), "the dog barks"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 3), "the"), new Annotation<>(new Span(4, 7), "dog"), From 4f264f957804cbf084d8c95d56d3fa1beb398c16 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 00:35:16 -0400 Subject: [PATCH 54/92] depparse: Validate the parsed graph size and pin the annotator's walk boundaries The annotator now rejects a parser that returns a graph over a different token count than its sentence, instead of silently misaligning the dependency layer with the token layer, and the javadoc states the text-order requirement the walk has always relied on. New tests pin the empty-sentence index shift, a token straddling two sentence spans, the stuck-scan path behind a gap token, and the graph-size rejection. The staged copy of the document container was refreshed to the current foundation, whose adapters parse per sentence and whose empty-versus-missing layer distinction moves the empty-text failure into this annotator's own validation. --- .../tools/depparse/DependencyAnnotator.java | 22 ++- .../DependencyAnnotatorEdgeCaseTest.java | 130 +++++++++++++++++- .../DependencyAnnotatorPipelineTest.java | 9 +- 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index dcc5aebd35..eb7db91e60 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -82,16 +82,24 @@ public DependencyAnnotator(DependencyParser parser) { * position, and each arc annotation reuses the span of its dependent token. A * sentence containing no tokens contributes no arcs.

* + *

The sentence and token layers must both be in text order: the walk assigns each + * sentence the contiguous run of tokens its span encloses, so a token that appears + * before its sentence in the layer, or a token overlapping a sentence boundary, is + * reported as lying outside every sentence rather than being silently attached to a + * neighboring sentence.

+ * * @param document The document to annotate. Must not be {@code null} and must carry a - * non-empty {@link Layers#SENTENCES} layer, a non-empty - * {@link Layers#TOKENS} layer whose every token lies inside a - * sentence, and a {@link Layers#POS_TAGS} layer of equal size. + * non-empty {@link Layers#SENTENCES} layer, in text order, a + * non-empty {@link Layers#TOKENS} layer, in text order, whose every + * token lies inside a sentence, and a {@link Layers#POS_TAGS} layer + * of equal size. * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the * token layer is absent or empty, the tag layer does not have exactly one - * tag per token, the sentence layer is absent or empty, or a token lies - * outside every sentence. + * tag per token, the sentence layer is absent or empty, a token lies + * outside every sentence under the text-order walk, or the parser returns a + * graph whose size differs from its sentence's token count. */ @Override public Document annotate(Document document) { @@ -132,6 +140,10 @@ public Document annotate(Document document) { posTags[i] = tags.get(first + i).value(); } final DependencyGraph graph = parser.parse(words, posTags); + if (graph.size() != count) { + throw new IllegalArgumentException("parser returned a graph over " + graph.size() + + " tokens for a sentence of " + count); + } // The parser indexes within the sentence; shifting by the sentence's first token // position turns every head and dependent into a document-wide token index, and // anchoring each arc on its dependent token's span puts the arc in document diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index e1b33d2a3d..d0899f70eb 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -47,6 +47,22 @@ public class DependencyAnnotatorEdgeCaseTest { DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, new String[] {"root", "obj"}); + /** + * A parser stub that returns a flat tree of the requested size, for assertions that + * must get past the annotator's graph-size validation with sentences of any length. + */ + private static final DependencyParser SIZE_MATCHING = (tokens, tags) -> { + final int[] heads = new int[tokens.length]; + final String[] relations = new String[tokens.length]; + heads[0] = DependencyArc.ROOT_HEAD; + relations[0] = "root"; + for (int i = 1; i < heads.length; i++) { + heads[i] = 0; + relations[i] = "dep"; + } + return DependencyGraph.of(heads, relations); + }; + /** * Builds a document over the text {@code "ab cd"} carrying aligned two-entry token and * tag layers, mirroring what the upstream tokenizer and tagger annotators would produce. @@ -177,7 +193,7 @@ void testSentenceLayerProblemsAreRejected() { new Annotation<>(new Span(0, 2), "VB"), new Annotation<>(new Span(3, 5), "NN"))); final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, - () -> new DependencyAnnotator(FIXED).annotate(strayToken)); + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(strayToken)); assertEquals("token at [3..5) lies outside every sentence", stray.getMessage()); } @@ -188,4 +204,116 @@ void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { .add(new DependencyAnnotator(FIXED)); assertThrows(IllegalArgumentException.class, builder::build); } + + /** + * Verifies the javadoc-promised behavior for a sentence containing no tokens: it + * contributes no arcs and no parser call, and the token indices of the sentence + * after it still shift by the correct first-token position rather than by a count + * that includes the empty sentence. + */ + @Test + void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { + final DependencyParser oneTokenRoot = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + final Document document = Document.of("ab. ??? cd.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "ab."), + new Annotation<>(new Span(4, 7), "???"), + new Annotation<>(new Span(8, 11), "cd."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(8, 10), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(8, 10), "VB"))); + + final List> arcs = + new DependencyAnnotator(oneTokenRoot).annotate(document) + .get(DependencyAnnotator.DEPENDENCIES); + assertEquals(2, arcs.size()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), + arcs.get(1).value()); + assertEquals(new Span(8, 10), arcs.get(1).span()); + } + + /** + * Verifies the text-order walk on a token straddling two sentence spans: the token + * belongs to neither sentence under the enclosure rule, the scan sticks at it, and + * the annotator reports it as lying outside every sentence instead of silently + * attaching it to one of its neighbors. + */ + @Test + void testTokenStraddlingTwoSentencesIsRejected() { + final Document document = Document.of("ab cd ef") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 4), "ab c"), + new Annotation<>(new Span(4, 8), "d ef"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); + assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies the stuck-scan path: a gap token between sentences stops the walk, and + * the token-bearing sentence after the gap does not pull the scan forward past the + * stray token, which is still reported rather than skipped. + */ + @Test + void testGapTokenBeforeATokenBearingSentenceIsStillRejected() { + final Document document = Document.of("ab cd ef.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(6, 9), "ef."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); + assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies that a parser returning a wrong-size graph is rejected loudly instead of + * silently misaligning the dependency layer with the token layer: the fixed + * two-token stub meets a three-token sentence and the annotator names both counts. + */ + @Test + void testWrongSizeGraphFailsLoud() { + final Document document = Document.of("ab cd ef") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 8), "ab cd ef"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(document)); + assertEquals("parser returned a graph over 2 tokens for a sentence of 3", + e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 97991c2754..1adcfbb810 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -285,10 +285,13 @@ void testSingleTokenSentenceParsesToARootArc() { } @Test - void testTextWithZeroSentencesFailsBeforeTheDependencyAnnotatorRuns() { - // empty text yields no sentences and no tokens, so the tagger already fails loud + void testTextWithZeroSentencesFailsAtTheDependencyAnnotator() { + // empty text yields present-but-empty sentence and token layers, which the + // upstream annotators pass through under the empty-versus-missing distinction; + // the dependency annotator itself then refuses to parse an empty token layer final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> pipeline().analyze("")); - assertEquals("document lacks the required layer tokens", e.getMessage()); + assertEquals("document needs aligned tokens and pos layers", + e.getMessage()); } } From fcf1571c71e3a686fcce336bf9528ccd9ce98d2a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 12:29:23 -0400 Subject: [PATCH 55/92] depparse: Mint the dependencies layer key in the toolkit namespace --- .../java/opennlp/tools/depparse/DependencyAnnotator.java | 2 +- .../tools/depparse/DependencyAnnotatorEdgeCaseTest.java | 8 ++++---- .../tools/depparse/DependencyAnnotatorPipelineTest.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index eb7db91e60..dc8282c0ab 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -55,7 +55,7 @@ public class DependencyAnnotator implements DocumentAnnotator { * position, anchored on the dependent token's span. */ public static final LayerKey DEPENDENCIES = - LayerKey.of("dependencies", DependencyArc.class); + Layers.key("dependencies", DependencyArc.class); private final DependencyParser parser; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index d0899f70eb..13a217c844 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -88,7 +88,7 @@ void testEmptyTokenAndTagLayersAreRejected() { .with(Layers.POS_TAGS, List.of()); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(empty)); - assertEquals("document needs aligned tokens and pos layers", + assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", e.getMessage()); } @@ -103,7 +103,7 @@ void testMisalignedTagLayerIsRejected() { new Annotation<>(new Span(0, 2), "VB"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(misaligned)); - assertEquals("document needs aligned tokens and pos layers", + assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", e.getMessage()); } @@ -116,7 +116,7 @@ void testAnnotatingTwiceIsRejected() { // documents are immutable and layers are add-once: a second pass must not overwrite final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> annotator.annotate(once)); - assertEquals("layer is already present: dependencies", e.getMessage()); + assertEquals("layer is already present: opennlp:dependencies", e.getMessage()); } @Test @@ -181,7 +181,7 @@ void testSentenceLayerProblemsAreRejected() { new Annotation<>(new Span(3, 5), "NN"))); final IllegalArgumentException missing = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(noSentences)); - assertEquals("document needs a non-empty sentences layer", + assertEquals("document needs a non-empty opennlp:sentences layer", missing.getMessage()); final Document strayToken = Document.of("ab cd") diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 1adcfbb810..745ef4ef85 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -291,7 +291,7 @@ void testTextWithZeroSentencesFailsAtTheDependencyAnnotator() { // the dependency annotator itself then refuses to parse an empty token layer final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> pipeline().analyze("")); - assertEquals("document needs aligned tokens and pos layers", + assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", e.getMessage()); } } From f7a1d9ee429feaf007054b5211642637826b6b55 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:45:31 -0400 Subject: [PATCH 56/92] dependency: Document the dependency annotator with a mirror-tested example Add a DependencyAnnotator section to the dependency chapter citing DependencyAnnotatorPipelineTest. --- opennlp-docs/src/docbkx/dependency.xml | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index a09c1afa54..71ac5ae7e2 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -54,4 +54,31 @@ DependencyGraph graph = parser.parse( DependencyEvaluator. + +
+ DependencyAnnotator + + DependencyAnnotator adapts a DependencyParser to + the document pipeline. It requires sentence, token, and POS-tag layers, + parses each sentence separately, and shifts sentence-local arcs so every + head and dependent is an index into the document-wide token layer. Each + arc annotation sits on its dependent token's span in original text + coordinates. DependencyAnnotatorPipelineTest asserts the + behavior shown here. + > arcs = + document.get(DependencyAnnotator.DEPENDENCIES); +// one arc per token; second-sentence heads are document-wide indexes +// "she" is token 3 with head 4 ("eats"), span [15..18)]]> + + +
From 8aac58b20720d07266cabf435be05afd6861022f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:54:58 -0400 Subject: [PATCH 57/92] dependency: Align annotator programlisting CDATA with sibling listings Open the CDATA on its own line so the rendered code block has no leading blank line, matching the two listings above it and parser.xml. --- opennlp-docs/src/docbkx/dependency.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 71ac5ae7e2..833440cd92 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -65,8 +65,8 @@ DependencyGraph graph = parser.parse( arc annotation sits on its dependent token's span in original text coordinates. DependencyAnnotatorPipelineTest asserts the behavior shown here. - + Date: Tue, 28 Jul 2026 07:05:38 -0400 Subject: [PATCH 58/92] depparse: Address review: absent-versus-empty layers, javadoc and test cleanup - Check each required layer for presence on its own, so a document missing the sentence, token, or tag layer is rejected with a message that names the key that is absent instead of being folded into the alignment complaint. - Accept present-but-empty required layers: a document with no sentences and no tokens now yields a present-but-empty dependencies layer rather than an IllegalArgumentException, which is the empty-versus-absent distinction the rest of the container annotators already follow. - Extract the shared rejection prefix into a MISSING_LAYER constant so all three absence checks emit one message shape. - Restate the annotate() javadoc and its @throws list against the checks that are actually performed, and say explicitly that the required layers may be empty. - Trim commentary that only repeated the javadoc: the class-level narration about being the first graph-shaped layer, the unwrap-the-slice comment, and the document-coordinate tail of the index-shift comment. - Replace the three hand-rolled absent-layer assertions with a parameterized test over one document per required layer, asserting the message names that layer. - Add pinning tests for the null-document message and for the empty document producing a present-but-empty arc layer, and flip the pipeline test on empty text to assert the same pass-through instead of a failure. - Assert the exact message in testMissingLayersThrow rather than only the exception type. - Hoist the one-token parser stub to a ONE_TOKEN_ROOT constant shared by the two tests that had declared it inline, and extract the repeated STRAY_TOKEN and MISALIGNED expected messages into constants. - Drop the two docbook sentences that pointed readers at test class names, state the present-but-empty layer contract in the annotator section, and correct "indexes" to "indices" in the example comment. --- .../tools/depparse/DependencyAnnotator.java | 68 ++++----- .../DependencyAnnotatorEdgeCaseTest.java | 135 +++++++++++------- .../DependencyAnnotatorPipelineTest.java | 16 +-- .../depparse/DependencyAnnotatorTest.java | 3 +- opennlp-docs/src/docbkx/dependency.xml | 8 +- 5 files changed, 135 insertions(+), 95 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index dc8282c0ab..e7a7d555fa 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -33,18 +33,16 @@ * provides {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's * span. * - *

This is the first graph-shaped layer: an arc's {@link DependencyArc#head()} and - * {@link DependencyArc#dependent()} are indices into the token layer, following the - * container's rule that annotations reference each other by layer and index, never by - * object identity.

+ *

An arc's {@link DependencyArc#head()} and {@link DependencyArc#dependent()} are + * indices into the token layer, following the container's rule that annotations + * reference each other by layer and index, never by object identity.

* - *

Each sentence is parsed separately, the way the parser is trained, so every - * sentence gets its own tree and its own root arc. The sentence-local indices the - * parser produces are shifted by the sentence's first token position, which keeps every - * arc's head and dependent a position in the document-wide token layer. Because every - * token span already refers to the original document text, anchoring an arc on its - * dependent token's span puts the arc in document coordinates without further offset - * arithmetic.

+ *

Each sentence is parsed separately, the way the parser contract expects its input, + * so every sentence gets its own tree and its own root arc. The sentence-local indices + * the parser returns are shifted by the sentence's first token position, which keeps + * every arc's head and dependent a position in the document-wide token layer. Token + * spans already refer to the original document text, so anchoring an arc on its + * dependent token's span puts the arc in document coordinates.

* * @since 3.0.0 */ @@ -57,6 +55,9 @@ public class DependencyAnnotator implements DocumentAnnotator { public static final LayerKey DEPENDENCIES = Layers.key("dependencies", DependencyArc.class); + /** The message prefix of every absent-required-layer rejection in this adapter. */ + private static final String MISSING_LAYER = "document lacks the required layer "; + private final DependencyParser parser; /** @@ -79,8 +80,10 @@ public DependencyAnnotator(DependencyParser parser) { * passed to the parser with their tags as one sequence, and the resulting * sentence-local arcs are shifted by the sentence's first token position. Arcs are * emitted in token order, so the new layer is aligned with {@link Layers#TOKENS} by - * position, and each arc annotation reuses the span of its dependent token. A - * sentence containing no tokens contributes no arcs.

+ * position, and each arc annotation reuses the span of its dependent token. The + * required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty dependency layer, and a sentence + * containing no tokens contributes no arcs.

* *

The sentence and token layers must both be in text order: the walk assigns each * sentence the contiguous run of tokens its span encloses, so a token that appears @@ -88,35 +91,39 @@ public DependencyAnnotator(DependencyParser parser) { * reported as lying outside every sentence rather than being silently attached to a * neighboring sentence.

* - * @param document The document to annotate. Must not be {@code null} and must carry a - * non-empty {@link Layers#SENTENCES} layer, in text order, a - * non-empty {@link Layers#TOKENS} layer, in text order, whose every - * token lies inside a sentence, and a {@link Layers#POS_TAGS} layer - * of equal size. + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, in + * text order, and a {@link Layers#POS_TAGS} layer with exactly one + * tag per token, with every token lying inside a sentence. * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * token layer is absent or empty, the tag layer does not have exactly one - * tag per token, the sentence layer is absent or empty, a token lies - * outside every sentence under the text-order walk, or the parser returns a - * graph whose size differs from its sentence's token count. + * sentence layer, the token layer, or the tag layer is absent, the tag layer + * does not have exactly one tag per token, a token lies outside every + * sentence under the text-order walk, or the parser returns a graph whose + * size differs from its sentence's token count. */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } + if (!document.layers().contains(Layers.SENTENCES)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.SENTENCES); + } + if (!document.layers().contains(Layers.TOKENS)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.TOKENS); + } + if (!document.layers().contains(Layers.POS_TAGS)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.POS_TAGS); + } + final List> sentences = document.get(Layers.SENTENCES); final List> tokens = document.get(Layers.TOKENS); final List> tags = document.get(Layers.POS_TAGS); - if (tokens.isEmpty() || tags.size() != tokens.size()) { + if (tags.size() != tokens.size()) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } - final List> sentences = document.get(Layers.SENTENCES); - if (sentences.isEmpty()) { - throw new IllegalArgumentException( - "document needs a non-empty " + Layers.SENTENCES + " layer"); - } final List> arcs = new ArrayList<>(tokens.size()); // Walk the token layer once: both layers are in text order, so each sentence // consumes the contiguous run of tokens whose spans it encloses. @@ -132,7 +139,6 @@ public Document annotate(Document document) { if (count == 0) { continue; } - // unwrap the sentence's slice into the parallel arrays the parser expects final String[] words = new String[count]; final String[] posTags = new String[count]; for (int i = 0; i < count; i++) { @@ -145,9 +151,7 @@ public Document annotate(Document document) { + " tokens for a sentence of " + count); } // The parser indexes within the sentence; shifting by the sentence's first token - // position turns every head and dependent into a document-wide token index, and - // anchoring each arc on its dependent token's span puts the arc in document - // coordinates, since token spans refer to the original text. + // position turns every head and dependent into a document-wide token index. for (final DependencyArc arc : graph.arcs()) { final int head = arc.head() == DependencyArc.ROOT_HEAD ? DependencyArc.ROOT_HEAD : arc.head() + first; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index 13a217c844..fea7b8137c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -19,12 +19,17 @@ import java.util.List; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import opennlp.tools.document.Annotation; import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.LayerKey; import opennlp.tools.document.Layers; import opennlp.tools.util.Span; @@ -33,12 +38,19 @@ /** * Pins down the boundary behavior of {@link DependencyAnnotator}: the exact exception and - * message for empty and misaligned input layers, the immutability rule that a second - * annotation pass is rejected, and the exact {@code requires()} and {@code provides()} - * declarations the pipeline validation relies on. + * message for absent, empty, and misaligned input layers, the immutability rule that a + * second annotation pass is rejected, and the exact {@code requires()} and + * {@code provides()} declarations the pipeline validation relies on. */ public class DependencyAnnotatorEdgeCaseTest { + /** The rejection message for a token that no sentence encloses. */ + private static final String STRAY_TOKEN = "token at [3..5) lies outside every sentence"; + + /** The rejection message for a tag layer that does not have one tag per token. */ + private static final String MISALIGNED = + "document needs aligned opennlp:tokens and opennlp:pos layers"; + /** * A parser stub that returns a fixed two-token graph regardless of its input, so the * assertions in this class exercise only the annotator's own layer handling. @@ -47,6 +59,18 @@ public class DependencyAnnotatorEdgeCaseTest { DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, new String[] {"root", "obj"}); + /** + * A parser stub that accepts only one-token sentences and returns their single root + * arc, so a sentence slice of any other length fails the test loudly. + */ + private static final DependencyParser ONE_TOKEN_ROOT = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + /** * A parser stub that returns a flat tree of the requested size, for assertions that * must get past the annotator's graph-size validation with sentences of any length. @@ -80,22 +104,67 @@ private static Document twoTokens() { new Annotation<>(new Span(3, 5), "NN"))); } + /** + * Supplies one document per required layer, each missing exactly that layer, together + * with the key the rejection message must name. + * + * @return The documents and the expected layer key. Never {@code null}. + */ + private static Stream documentsMissingOneLayer() { + final List> sentence = + List.of(new Annotation<>(new Span(0, 5), "ab cd")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd")); + final List> tags = List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN")); + return Stream.of( + Arguments.of(Document.of("ab cd") + .with(Layers.TOKENS, tokens).with(Layers.POS_TAGS, tags), Layers.SENTENCES), + Arguments.of(Document.of("ab cd") + .with(Layers.SENTENCES, sentence).with(Layers.POS_TAGS, tags), Layers.TOKENS), + Arguments.of(Document.of("ab cd") + .with(Layers.SENTENCES, sentence).with(Layers.TOKENS, tokens), Layers.POS_TAGS)); + } + + @ParameterizedTest + @MethodSource("documentsMissingOneLayer") + void testAbsentRequiredLayerIsNamed(Document document, LayerKey missing) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(document)); + assertEquals("document lacks the required layer " + missing, e.getMessage()); + } + @Test - void testEmptyTokenAndTagLayersAreRejected() { - // a document with zero sentences has zero tokens; the annotator refuses to parse it + void testNullDocumentIsRejected() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(null)); + assertEquals("document must not be null", e.getMessage()); + } + + /** + * Verifies the empty-versus-absent distinction of the annotator contract: present but + * empty required layers are valid input and yield a present-but-empty arc layer, so a + * pipeline does not fail on a document without content. + */ + @Test + void testEmptyRequiredLayersYieldAnEmptyArcLayer() { final Document empty = Document.of("") + .with(Layers.SENTENCES, List.of()) .with(Layers.TOKENS, List.of()) .with(Layers.POS_TAGS, List.of()); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> new DependencyAnnotator(FIXED).annotate(empty)); - assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", - e.getMessage()); + final Document annotated = new DependencyAnnotator(FIXED).annotate(empty); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, + DependencyAnnotator.DEPENDENCIES), annotated.layers()); + assertEquals(List.of(), annotated.get(DependencyAnnotator.DEPENDENCIES)); } @Test void testMisalignedTagLayerIsRejected() { // two tokens but only one tag: the layers are present yet not aligned by position final Document misaligned = Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "ab cd"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 2), "ab"), new Annotation<>(new Span(3, 5), "cd"))) @@ -103,8 +172,7 @@ void testMisalignedTagLayerIsRejected() { new Annotation<>(new Span(0, 2), "VB"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(misaligned)); - assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", - e.getMessage()); + assertEquals(MISALIGNED, e.getMessage()); } @Test @@ -134,14 +202,6 @@ void testRequiresAndProvidesDeclarationsAreExact() { */ @Test void testEachSentenceGetsItsOwnTree() { - final DependencyParser oneTokenRoot = (tokens, tags) -> { - if (tokens.length != 1) { - throw new IllegalStateException("expected one-token sentences, got " - + tokens.length); - } - return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, - new String[] {"root"}); - }; final Document document = Document.of("ab. cd.") .with(Layers.SENTENCES, List.of( new Annotation<>(new Span(0, 3), "ab."), @@ -154,7 +214,7 @@ void testEachSentenceGetsItsOwnTree() { new Annotation<>(new Span(4, 6), "VB"))); final List> arcs = - new DependencyAnnotator(oneTokenRoot).annotate(document) + new DependencyAnnotator(ONE_TOKEN_ROOT).annotate(document) .get(DependencyAnnotator.DEPENDENCIES); assertEquals(2, arcs.size()); assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 0, "root"), @@ -166,24 +226,11 @@ void testEachSentenceGetsItsOwnTree() { } /** - * Verifies the sentence-layer requirements fail loud: a token-bearing document - * without a sentence layer is rejected, and so is a token lying outside every - * sentence. + * Verifies a token that no sentence encloses is reported instead of being parsed + * outside of any sentence. */ @Test - void testSentenceLayerProblemsAreRejected() { - final Document noSentences = Document.of("ab cd") - .with(Layers.TOKENS, List.of( - new Annotation<>(new Span(0, 2), "ab"), - new Annotation<>(new Span(3, 5), "cd"))) - .with(Layers.POS_TAGS, List.of( - new Annotation<>(new Span(0, 2), "VB"), - new Annotation<>(new Span(3, 5), "NN"))); - final IllegalArgumentException missing = assertThrows(IllegalArgumentException.class, - () -> new DependencyAnnotator(FIXED).annotate(noSentences)); - assertEquals("document needs a non-empty opennlp:sentences layer", - missing.getMessage()); - + void testTokenOutsideEverySentenceIsRejected() { final Document strayToken = Document.of("ab cd") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 2), "ab"))) .with(Layers.TOKENS, List.of( @@ -194,7 +241,7 @@ void testSentenceLayerProblemsAreRejected() { new Annotation<>(new Span(3, 5), "NN"))); final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(SIZE_MATCHING).annotate(strayToken)); - assertEquals("token at [3..5) lies outside every sentence", stray.getMessage()); + assertEquals(STRAY_TOKEN, stray.getMessage()); } @Test @@ -213,14 +260,6 @@ void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { */ @Test void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { - final DependencyParser oneTokenRoot = (tokens, tags) -> { - if (tokens.length != 1) { - throw new IllegalStateException("expected one-token sentences, got " - + tokens.length); - } - return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, - new String[] {"root"}); - }; final Document document = Document.of("ab. ??? cd.") .with(Layers.SENTENCES, List.of( new Annotation<>(new Span(0, 3), "ab."), @@ -234,7 +273,7 @@ void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { new Annotation<>(new Span(8, 10), "VB"))); final List> arcs = - new DependencyAnnotator(oneTokenRoot).annotate(document) + new DependencyAnnotator(ONE_TOKEN_ROOT).annotate(document) .get(DependencyAnnotator.DEPENDENCIES); assertEquals(2, arcs.size()); assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), @@ -265,7 +304,7 @@ void testTokenStraddlingTwoSentencesIsRejected() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); - assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + assertEquals(STRAY_TOKEN, e.getMessage()); } /** @@ -290,7 +329,7 @@ void testGapTokenBeforeATokenBearingSentenceIsStillRejected() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); - assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + assertEquals(STRAY_TOKEN, e.getMessage()); } /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 745ef4ef85..2147c223f4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -42,7 +42,6 @@ import opennlp.tools.util.TrainingParameters; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; /** * Demonstrates {@link DependencyAnnotator} at the end of a complete {@link DocumentAnalyzer} @@ -285,13 +284,12 @@ void testSingleTokenSentenceParsesToARootArc() { } @Test - void testTextWithZeroSentencesFailsAtTheDependencyAnnotator() { - // empty text yields present-but-empty sentence and token layers, which the - // upstream annotators pass through under the empty-versus-missing distinction; - // the dependency annotator itself then refuses to parse an empty token layer - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> pipeline().analyze("")); - assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", - e.getMessage()); + void testTextWithZeroSentencesYieldsAnEmptyDependencyLayer() { + // empty text yields present-but-empty sentence, token, and tag layers, which every + // annotator of the pipeline passes through under the empty-versus-absent + // distinction, so the dependency layer comes out present and empty + final Document document = pipeline().analyze(""); + assertEquals(List.of(), document.get(Layers.TOKENS)); + assertEquals(List.of(), document.get(DependencyAnnotator.DEPENDENCIES)); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java index 554cb3a72d..c5b215452b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -97,8 +97,9 @@ void testRootArcCarriesRootHead() { @Test void testMissingLayersThrow() { final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); - assertThrows(IllegalArgumentException.class, + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no layers"))); + assertEquals("document lacks the required layer " + Layers.SENTENCES, e.getMessage()); } @Test diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 833440cd92..46d01ce2ac 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -24,8 +24,6 @@ DependencyParserME trains on DependencySample streams and returns a DependencyGraph of heads and relations. CoNLL-U treebanks are read through ConlluDependencySampleStream. - ConlluDependencyParserUsageTest asserts the train-parse-evaluate - workflow shown here. @@ -63,8 +61,8 @@ DependencyGraph graph = parser.parse( parses each sentence separately, and shifts sentence-local arcs so every head and dependent is an index into the document-wide token layer. Each arc annotation sits on its dependent token's span in original text - coordinates. DependencyAnnotatorPipelineTest asserts the - behavior shown here. + coordinates. The required layers must be present, but they may be empty: + a document without content yields a present-but-empty arc layer. > arcs = document.get(DependencyAnnotator.DEPENDENCIES); -// one arc per token; second-sentence heads are document-wide indexes +// one arc per token; second-sentence heads are document-wide indices // "she" is token 3 with head 4 ("eats"), span [15..18)]]> From d17b74a6b9f189967aacaa405c7a444f92eb3810 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:56:08 -0400 Subject: [PATCH 59/92] dependency: Cite the annotator pipeline test in the manual section --- opennlp-docs/src/docbkx/dependency.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 46d01ce2ac..a4e8d31f38 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -63,6 +63,7 @@ DependencyGraph graph = parser.parse( arc annotation sits on its dependent token's span in original text coordinates. The required layers must be present, but they may be empty: a document without content yields a present-but-empty arc layer. + DependencyAnnotatorPipelineTest asserts the behavior shown here. Date: Wed, 26 Aug 2026 05:41:02 -0400 Subject: [PATCH 60/92] OPENNLP-1888: Move concrete annotators to runtime --- .../src/main/java/opennlp/tools/document/NameFinderAnnotator.java | 0 .../src/main/java/opennlp/tools/document/POSTaggerAnnotator.java | 0 .../java/opennlp/tools/document/SentenceDetectorAnnotator.java | 0 .../src/main/java/opennlp/tools/document/TokenizerAnnotator.java | 0 .../test/java/opennlp/tools/document/DocumentAnalyzerTest.java | 0 .../java/opennlp/tools/document/DocumentPipelineExampleTest.java | 0 .../test/java/opennlp/tools/document/NameFinderAnnotatorTest.java | 0 .../test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java | 0 .../src/test/java/opennlp/tools/document/TestComponents.java | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/document/NameFinderAnnotator.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/document/TokenizerAnnotator.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/document/TestComponents.java (100%) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/NameFinderAnnotator.java similarity index 100% rename from opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/NameFinderAnnotator.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java similarity index 100% rename from opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java similarity index 100% rename from opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/TokenizerAnnotator.java similarity index 100% rename from opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/TokenizerAnnotator.java diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java diff --git a/opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java diff --git a/opennlp-api/src/test/java/opennlp/tools/document/TestComponents.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/document/TestComponents.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java From 22780da3da0d219564050dc30a0ff333de35a441 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 26 Aug 2026 07:02:48 -0400 Subject: [PATCH 61/92] OPENNLP-1888: Place annotators with their components --- .../tools/document/DocumentContractTest.java | 28 +++++++++++++++ .../NameFinderAnnotator.java | 10 ++++-- .../POSTaggerAnnotator.java | 9 +++-- .../SentenceDetectorAnnotator.java | 8 +++-- .../TokenizerAnnotator.java | 8 +++-- .../tools/document/DocumentAnalyzerTest.java | 34 +++---------------- .../document/DocumentPipelineExampleTest.java | 3 ++ .../NameFinderAnnotatorTest.java | 7 ++-- .../POSTaggerAnnotatorTest.java | 6 ++-- 9 files changed, 69 insertions(+), 44 deletions(-) rename opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/{document => namefind}/NameFinderAnnotator.java (95%) rename opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/{document => postag}/POSTaggerAnnotator.java (93%) rename opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/{document => sentdetect}/SentenceDetectorAnnotator.java (92%) rename opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/{document => tokenize}/TokenizerAnnotator.java (94%) rename opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/{document => namefind}/NameFinderAnnotatorTest.java (98%) rename opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/{document => postag}/POSTaggerAnnotatorTest.java (98%) diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java index e83a01508c..27a6a1948f 100644 --- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -384,6 +384,34 @@ void testEmptyPipelineFailsWithExactMessage() { assertEquals("a pipeline needs at least one annotator", e.getMessage()); } + /** + * Verifies that a caller can add a new layer type without changing the document + * container. + */ + @Test + void testCustomLayerNeedsNoContainerChange() { + record Sentiment(String polarity, double score) { + } + final LayerKey sentiment = LayerKey.of("sentiment", Sentiment.class); + final DocumentAnnotator annotator = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + final Span all = new Span(0, document.text().length()); + return document.with(sentiment, + List.of(new Annotation<>(all, new Sentiment("positive", 0.9d)))); + } + + @Override + public Set> provides() { + return Set.of(sentiment); + } + }; + final Document document = DocumentAnalyzer.builder().add(annotator).build() + .analyze("good dog"); + assertEquals("positive", document.get(sentiment).get(0).value().polarity()); + } + /** * Verifies that the value type travels through {@link LayerKey}: a layer added under * an {@code Integer} key reads back as {@code Annotation}, so its values diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java similarity index 95% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/NameFinderAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java index 97e9d9526d..43baa231e0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/NameFinderAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java @@ -15,14 +15,18 @@ * limitations under the License. */ -package opennlp.tools.document; +package opennlp.tools.namefind; import java.util.ArrayList; import java.util.List; import java.util.Set; -import opennlp.tools.namefind.NameSample; -import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; import opennlp.tools.util.Span; /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java similarity index 93% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java index 82facfba2b..feb34b6aa4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java @@ -15,13 +15,18 @@ * limitations under the License. */ -package opennlp.tools.document; +package opennlp.tools.postag; import java.util.ArrayList; import java.util.List; import java.util.Set; -import opennlp.tools.postag.POSTagger; +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; /** * Adapts a {@link POSTagger} to the document pipeline: reads {@link Layers#SENTENCES} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java similarity index 92% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java index 183a6df13e..0586daedc3 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java @@ -15,13 +15,17 @@ * limitations under the License. */ -package opennlp.tools.document; +package opennlp.tools.sentdetect; import java.util.ArrayList; import java.util.List; import java.util.Set; -import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; import opennlp.tools.util.Span; /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java similarity index 94% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/TokenizerAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java index b7d6b9e288..7760b565d3 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/document/TokenizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java @@ -15,13 +15,17 @@ * limitations under the License. */ -package opennlp.tools.document; +package opennlp.tools.tokenize; import java.util.ArrayList; import java.util.List; import java.util.Set; -import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; import opennlp.tools.util.Span; /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java index b6d9234b66..7af7a75162 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -24,8 +24,12 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.namefind.NameFinderAnnotator; import opennlp.tools.namefind.TokenNameFinder; import opennlp.tools.postag.POSTagger; +import opennlp.tools.postag.POSTaggerAnnotator; +import opennlp.tools.sentdetect.SentenceDetectorAnnotator; +import opennlp.tools.tokenize.TokenizerAnnotator; import opennlp.tools.util.Sequence; import opennlp.tools.util.Span; @@ -173,36 +177,6 @@ void testMisorderedPipelineFailsAtBuildTime() { assertThrows(IllegalArgumentException.class, builder::build); } - @Test - void testEmptyPipelineThrows() { - assertThrows(IllegalArgumentException.class, () -> DocumentAnalyzer.builder().build()); - } - - @Test - void testCustomLayerNeedsNoContainerChange() { - // the additive claim: a brand-new layer type works without touching the container - record Sentiment(String polarity, double score) { - } - final LayerKey sentiment = LayerKey.of("sentiment", Sentiment.class); - final DocumentAnnotator annotator = new DocumentAnnotator() { - - @Override - public Document annotate(Document document) { - final Span all = new Span(0, document.text().length()); - return document.with(sentiment, - List.of(new Annotation<>(all, new Sentiment("positive", 0.9d)))); - } - - @Override - public Set> provides() { - return Set.of(sentiment); - } - }; - final Document document = DocumentAnalyzer.builder().add(annotator).build() - .analyze("good dog"); - assertEquals("positive", document.get(sentiment).get(0).value().polarity()); - } - @Test void testAnnotatorAdaptersRejectNullDelegates() { assertThrows(IllegalArgumentException.class, () -> new SentenceDetectorAnnotator(null)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java index 0595b53716..f38737d668 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java @@ -25,6 +25,9 @@ import org.junit.jupiter.api.Test; import opennlp.tools.postag.POSTagger; +import opennlp.tools.postag.POSTaggerAnnotator; +import opennlp.tools.sentdetect.SentenceDetectorAnnotator; +import opennlp.tools.tokenize.TokenizerAnnotator; import opennlp.tools.util.Sequence; import opennlp.tools.util.Span; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderAnnotatorTest.java similarity index 98% rename from opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderAnnotatorTest.java index e46be8bb2f..14f294c8ce 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderAnnotatorTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package opennlp.tools.document; +package opennlp.tools.namefind; import java.util.ArrayList; import java.util.List; @@ -25,8 +25,9 @@ import org.junit.jupiter.api.Test; -import opennlp.tools.namefind.NameSample; -import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; import opennlp.tools.util.Span; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSTaggerAnnotatorTest.java similarity index 98% rename from opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSTaggerAnnotatorTest.java index ccd7b0cfe0..72fea7f524 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/POSTaggerAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSTaggerAnnotatorTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package opennlp.tools.document; +package opennlp.tools.postag; import java.util.ArrayList; import java.util.Arrays; @@ -24,7 +24,9 @@ import org.junit.jupiter.api.Test; -import opennlp.tools.postag.POSTagger; +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; import opennlp.tools.util.Sequence; import opennlp.tools.util.Span; From 5115aaa402cf3a71b067246b5a68172fd75e365f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 14 Jul 2026 21:09:58 -0400 Subject: [PATCH 62/92] depparse: Arc-standard dependency parser skeleton with CoNLL-U reader and UAS/LAS evaluator Adds opennlp.tools.depparse: the DependencyParser interface with DependencyArc, DependencyGraph, and DependencySample in opennlp-api, and a greedy transition-based implementation in the runtime: arc-standard state and static oracle, configuration features, an oracle-driven event stream, DependencyParserME over the existing maxent machinery, and a UAS/LAS evaluator. ConlluDependencySampleStream maps the basic dependency columns of CoNLL-U sentences so Universal Dependencies treebanks train directly; sentences whose multiword tokens were merged by ConlluStream are skipped because their syntactic words are not recoverable. Non-projective trees have no arc-standard derivation and are skipped during event generation. Model persistence and a DL-backed implementation are follow-ups behind the same interface. (cherry picked from commit 7448e34080480cc52b266069f5c700557957085c) --- .../opennlp/tools/depparse/DependencyArc.java | 63 +++++ .../tools/depparse/DependencyGraph.java | 179 ++++++++++++++ .../tools/depparse/DependencyParser.java | 47 ++++ .../tools/depparse/DependencySample.java | 113 +++++++++ .../tools/depparse/DependencyGraphTest.java | 140 +++++++++++ .../tools/depparse/DependencySampleTest.java | 78 +++++++ .../conllu/ConlluDependencySampleStream.java | 131 +++++++++++ .../ConlluDependencySampleStreamTest.java | 105 +++++++++ .../tools/depparse/ArcStandardOracle.java | 104 +++++++++ .../tools/depparse/ArcStandardState.java | 219 ++++++++++++++++++ .../depparse/DependencyContextGenerator.java | 115 +++++++++ .../tools/depparse/DependencyEvaluator.java | 88 +++++++ .../tools/depparse/DependencyEventStream.java | 107 +++++++++ .../tools/depparse/DependencyParserME.java | 140 +++++++++++ .../opennlp/tools/depparse/Transition.java | 127 ++++++++++ .../tools/depparse/ArcStandardOracleTest.java | 87 +++++++ .../depparse/DependencyParserMETest.java | 122 ++++++++++ 17 files changed, 1965 insertions(+) create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java create mode 100644 opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java new file mode 100644 index 0000000000..bf9dc2eea4 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * One labeled edge of a {@link DependencyGraph}: the token at {@code dependent} is governed + * by the token at {@code head} under the given {@code relation}. + * + *

Indices are zero-based positions in the token array the graph was built over. A + * {@code head} of {@link #ROOT_HEAD} marks the dependent as the sentence root, which is + * attached to the artificial root node rather than to another token.

+ * + * @param head The zero-based index of the governing token, or {@link #ROOT_HEAD} when the + * dependent is the sentence root. + * @param dependent The zero-based index of the governed token. + * @param relation The dependency relation label, for example {@code nsubj}. + * + * @since 3.0.0 + */ +public record DependencyArc(int head, int dependent, String relation) { + + /** + * The {@code head} value marking an arc from the artificial root node. + */ + public static final int ROOT_HEAD = -1; + + /** + * Validates the arc invariants. + * + * @throws IllegalArgumentException Thrown if {@code dependent} is negative, {@code head} + * is less than {@link #ROOT_HEAD}, the arc is a self-loop, or {@code relation} + * is {@code null} or blank. + */ + public DependencyArc { + if (dependent < 0) { + throw new IllegalArgumentException("dependent must not be negative: " + dependent); + } + if (head < ROOT_HEAD) { + throw new IllegalArgumentException("head must be a token index or ROOT_HEAD: " + head); + } + if (head == dependent) { + throw new IllegalArgumentException("arc must not be a self-loop: " + head); + } + if (relation == null || relation.isBlank()) { + throw new IllegalArgumentException("relation must not be null or blank"); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java new file mode 100644 index 0000000000..e95234dcc8 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * An immutable dependency tree over one sentence: for every token, the index of its head + * and the label of the relation to that head. + * + *

Token indices are zero-based positions in the sentence the graph was built for. + * Exactly one token carries the head value {@link DependencyArc#ROOT_HEAD}, marking it as + * the sentence root. Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public final class DependencyGraph { + + private final int[] heads; + private final String[] relations; + + private DependencyGraph(int[] heads, String[] relations) { + this.heads = heads; + this.relations = relations; + } + + /** + * Creates a {@link DependencyGraph} from parallel head and relation arrays. + * + * @param heads For each token, the zero-based index of its head token, or + * {@link DependencyArc#ROOT_HEAD} for the sentence root. Must not be + * {@code null} or empty, every value must be a valid token index or + * {@link DependencyArc#ROOT_HEAD}, no token may head itself, and exactly + * one token must be the root. + * @param relations For each token, the label of the relation to its head. Must not be + * {@code null}, must have the same length as {@code heads}, and no + * entry may be {@code null} or blank. + * @return A validated {@link DependencyGraph}. Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + public static DependencyGraph of(int[] heads, String[] relations) { + if (heads == null || relations == null) { + throw new IllegalArgumentException("heads and relations must not be null"); + } + if (heads.length == 0) { + throw new IllegalArgumentException("a dependency graph needs at least one token"); + } + if (heads.length != relations.length) { + throw new IllegalArgumentException("heads and relations must have the same length: " + + heads.length + " != " + relations.length); + } + int roots = 0; + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + roots++; + } else if (heads[i] < 0 || heads[i] >= heads.length) { + throw new IllegalArgumentException("head of token " + i + + " is out of range: " + heads[i]); + } else if (heads[i] == i) { + throw new IllegalArgumentException("token " + i + " must not head itself"); + } + if (relations[i] == null || relations[i].isBlank()) { + throw new IllegalArgumentException("relation of token " + i + " must not be blank"); + } + } + if (roots != 1) { + throw new IllegalArgumentException("expected exactly one root, found " + roots); + } + return new DependencyGraph(heads.clone(), relations.clone()); + } + + /** + * @return The number of tokens the graph spans. + */ + public int size() { + return heads.length; + } + + /** + * Retrieves the head of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The zero-based index of the head token, or {@link DependencyArc#ROOT_HEAD} + * when the token is the sentence root. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int headOf(int index) { + checkIndex(index); + return heads[index]; + } + + /** + * Retrieves the relation label of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The label of the relation between the token and its head. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String relationOf(int index) { + checkIndex(index); + return relations[index]; + } + + /** + * @return The zero-based index of the sentence root token. + */ + public int root() { + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + return i; + } + } + throw new IllegalStateException("graph invariant violated: no root present"); + } + + /** + * @return All arcs of the graph in token order, one per token. Never {@code null}. + */ + public List arcs() { + final List arcs = new ArrayList<>(heads.length); + for (int i = 0; i < heads.length; i++) { + arcs.add(new DependencyArc(heads[i], i, relations[i])); + } + return Collections.unmodifiableList(arcs); + } + + private void checkIndex(int index) { + if (index < 0 || index >= heads.length) { + throw new IllegalArgumentException("token index out of range: " + index + + ", size: " + heads.length); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencyGraph other)) { + return false; + } + return Arrays.equals(heads, other.heads) && Arrays.equals(relations, other.relations); + } + + @Override + public int hashCode() { + return 31 * Arrays.hashCode(heads) + Arrays.hashCode(relations); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < heads.length; i++) { + if (i > 0) { + sb.append(' '); + } + sb.append(i).append("<-").append(heads[i]).append(':').append(relations[i]); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java new file mode 100644 index 0000000000..0c6e5d1303 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The interface for dependency parsers, which assign every token of a sentence a syntactic + * head and a relation label, forming a single-rooted tree over the sentence. + * + *

Dependency parsing complements the constituency {@link opennlp.tools.parser.Parser}: + * where a constituency parse groups tokens into nested phrases, a dependency parse links + * each token directly to the token it modifies. The result is a {@link DependencyGraph} + * whose indices refer back to the input token array, so spans computed for those tokens + * remain valid for the parse.

+ * + * @see DependencyGraph + * @since 3.0.0 + */ +public interface DependencyParser { + + /** + * Parses a sentence into a {@link DependencyGraph}. + * + * @param tokens The tokens of one sentence. Must not be {@code null} and must contain + * at least one token. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must have the same length as {@code tokens}. + * @return A {@link DependencyGraph} over the given tokens. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokens} or {@code tags} is + * {@code null}, empty, or of mismatched length. + */ + DependencyGraph parse(String[] tokens, String[] tags); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java new file mode 100644 index 0000000000..b805e79a51 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.Arrays; +import java.util.Objects; + +/** + * One dependency-annotated sentence: tokens, their part-of-speech tags, and the gold + * {@link DependencyGraph} over them. Used for training and evaluating a + * {@link DependencyParser}. + * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public class DependencySample { + + private final String[] tokens; + private final String[] tags; + private final DependencyGraph graph; + + /** + * Initializes a {@link DependencySample}. + * + * @param tokens The tokens of the sentence. Must not be {@code null} or empty. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must have the same length as {@code tokens}. + * @param graph The dependency graph over the tokens. Must not be {@code null} and its + * {@link DependencyGraph#size()} must equal the number of tokens. + * @throws IllegalArgumentException Thrown if any parameter is {@code null} or the + * lengths disagree. + */ + public DependencySample(String[] tokens, String[] tags, DependencyGraph graph) { + if (tokens == null || tags == null || graph == null) { + throw new IllegalArgumentException("tokens, tags and graph must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("a sample needs at least one token"); + } + if (tokens.length != tags.length || tokens.length != graph.size()) { + throw new IllegalArgumentException("tokens, tags and graph must agree in length: " + + tokens.length + ", " + tags.length + ", " + graph.size()); + } + this.tokens = tokens.clone(); + this.tags = tags.clone(); + this.graph = graph; + } + + /** + * @return The tokens of the sentence. Never {@code null}. + */ + public String[] getTokens() { + return tokens.clone(); + } + + /** + * @return The part-of-speech tags aligned with the tokens. Never {@code null}. + */ + public String[] getTags() { + return tags.clone(); + } + + /** + * @return The dependency graph over the tokens. Never {@code null}. + */ + public DependencyGraph getGraph() { + return graph; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencySample other)) { + return false; + } + return Arrays.equals(tokens, other.tokens) && Arrays.equals(tags, other.tags) + && graph.equals(other.graph); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(tokens), Arrays.hashCode(tags), graph); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < tokens.length; i++) { + sb.append(i + 1).append('\t').append(tokens[i]).append('\t').append(tags[i]) + .append('\t').append(graph.headOf(i) + 1).append('\t').append(graph.relationOf(i)) + .append(System.lineSeparator()); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java new file mode 100644 index 0000000000..91bf86bd19 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants and accessors of {@link DependencyGraph} and {@link DependencyArc}. + */ +public class DependencyGraphTest { + + private static DependencyGraph sample() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencyGraph graph = sample(); + assertEquals(3, graph.size()); + assertEquals(1, graph.headOf(0)); + assertEquals(2, graph.headOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, graph.headOf(2)); + assertEquals("nsubj", graph.relationOf(1)); + assertEquals(2, graph.root()); + } + + @Test + void testArcsAreInTokenOrder() { + final List arcs = sample().arcs(); + assertEquals(3, arcs.size()); + assertEquals(new DependencyArc(1, 0, "det"), arcs.get(0)); + assertEquals(new DependencyArc(2, 1, "nsubj"), arcs.get(1)); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 2, "root"), arcs.get(2)); + } + + @Test + void testEqualsAndHashCode() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + assertNotEquals(sample(), DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"amod", "nsubj", "root"})); + } + + @Test + void testInputArraysAreCopied() { + final int[] heads = {1, -1}; + final String[] relations = {"nsubj", "root"}; + final DependencyGraph graph = DependencyGraph.of(heads, relations); + heads[0] = 0; + relations[0] = "det"; + assertEquals(1, graph.headOf(0)); + assertEquals("nsubj", graph.relationOf(0)); + } + + @Test + void testNullArraysThrow() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(null, new String[] {"root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, null)); + } + + @Test + void testEmptyGraphThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[0], new String[0])); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, new String[] {"root", "nsubj"})); + } + + @Test + void testRootCountIsEnforced() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, 0}, new String[] {"a", "b"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1, -1}, new String[] {"root", "root"})); + } + + @Test + void testSelfHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {0, -1}, new String[] {"a", "root"})); + } + + @Test + void testOutOfRangeHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {2, -1}, new String[] {"a", "root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-3, -1}, new String[] {"a", "root"})); + } + + @Test + void testBlankRelationThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {" ", "root"})); + } + + @Test + void testIndexBoundsThrow() { + final DependencyGraph graph = sample(); + assertThrows(IllegalArgumentException.class, () -> graph.headOf(-1)); + assertThrows(IllegalArgumentException.class, () -> graph.relationOf(3)); + } + + @Test + void testArcValidation() { + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(0, 0, "root")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, -1, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(-2, 0, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, " ")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java new file mode 100644 index 0000000000..03234eab9f --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants of {@link DependencySample}. + */ +public class DependencySampleTest { + + private static final String[] TOKENS = {"the", "dog", "barks"}; + private static final String[] TAGS = {"DT", "NN", "VBZ"}; + + private static DependencyGraph graph() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencySample sample = new DependencySample(TOKENS, TAGS, graph()); + assertArrayEquals(TOKENS, sample.getTokens()); + assertArrayEquals(TAGS, sample.getTags()); + assertEquals(graph(), sample.getGraph()); + } + + @Test + void testEquals() { + assertEquals(new DependencySample(TOKENS, TAGS, graph()), + new DependencySample(TOKENS, TAGS, graph())); + } + + @Test + void testNullArgumentsThrow() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(null, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, null, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, TAGS, null)); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[] {"one"}, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, new String[] {"DT"}, graph())); + } + + @Test + void testInputArraysAreCopied() { + final String[] tokens = TOKENS.clone(); + final DependencySample sample = new DependencySample(tokens, TAGS, graph()); + tokens[0] = "a"; + assertEquals("the", sample.getTokens()[0]); + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java new file mode 100644 index 0000000000..0dae3f7fc6 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.FilterObjectStream; +import opennlp.tools.util.ObjectStream; + +/** + * Reads {@link DependencySample samples} from a stream of {@link ConlluSentence sentences}, + * mapping the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * + *

Empty nodes carry no basic dependency annotation and are dropped; the remaining word + * lines keep their one-based ids, which are shifted to the zero-based indices of + * {@link DependencyGraph}. Sentences containing a multiword token are skipped entirely: + * {@link ConlluStream} merges such a range with its syntactic words, so the dependency + * annotation of those words is no longer recoverable. Sentences whose annotation is + * incomplete or invalid, for example an underscore head, are skipped as well. Skips are + * counted and the count is logged once the stream is exhausted.

+ * + * @since 3.0.0 + */ +public class ConlluDependencySampleStream + extends FilterObjectStream { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencySampleStream.class); + + private final ConlluTagset tagset; + + private int skipped; + + /** + * Initializes the stream. + * + * @param samples The sentences to convert. Must not be {@code null}. + * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must not + * be {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public ConlluDependencySampleStream(ObjectStream samples, + ConlluTagset tagset) { + super(samples); + if (tagset == null) { + throw new IllegalArgumentException("tagset must not be null"); + } + this.tagset = tagset; + } + + @Override + public DependencySample read() throws IOException { + ConlluSentence sentence; + while ((sentence = samples.read()) != null) { + final DependencySample sample = convert(sentence); + if (sample != null) { + return sample; + } + skipped++; + } + if (skipped > 0) { + logger.warn("Skipped {} sentence(s) without a complete basic dependency annotation.", + skipped); + skipped = 0; + } + return null; + } + + /** + * Converts one sentence, or returns {@code null} when its annotation is unusable. + */ + private DependencySample convert(ConlluSentence sentence) { + final List words = new ArrayList<>(); + for (final ConlluWordLine line : sentence.getWordLines()) { + final String id = line.getId(); + if (id.indexOf('-') >= 0) { + // a merged multiword token: its syntactic words are gone, the sentence is unusable + return null; + } + if (id.indexOf('.') < 0) { + words.add(line); + } + } + if (words.isEmpty()) { + return null; + } + final int n = words.size(); + final String[] tokens = new String[n]; + final String[] tags = new String[n]; + final int[] heads = new int[n]; + final String[] relations = new String[n]; + for (int i = 0; i < n; i++) { + final ConlluWordLine word = words.get(i); + tokens[i] = word.getForm(); + tags[i] = word.getPosTag(tagset); + relations[i] = word.getDeprel(); + try { + heads[i] = Integer.parseInt(word.getHead()) - 1; + } catch (NumberFormatException e) { + return null; + } + } + try { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } catch (IllegalArgumentException e) { + return null; + } + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java new file mode 100644 index 0000000000..6105efd00e --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencySample; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests that {@link ConlluDependencySampleStream} maps the basic dependency columns and + * skips sentences without a usable annotation, including sentences whose multiword tokens + * were merged by {@link ConlluStream}. + */ +public class ConlluDependencySampleStreamTest { + + private static String line(String... fields) { + return String.join("\t", fields); + } + + private static final String CONLLU = String.join("\n", + "# sent_id = test-1", + "# text = He bought the bonds", + line("1", "He", "he", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "bought", "buy", "VERB", "VBD", "_", "0", "root", "_", "_"), + line("3", "the", "the", "DET", "DT", "_", "4", "det", "_", "_"), + line("4", "bonds", "bond", "NOUN", "NNS", "_", "2", "obj", "_", "_"), + "", + "# sent_id = test-2", + "# text = Broken", + line("1", "Broken", "broken", "ADJ", "JJ", "_", "_", "_", "_", "_"), + "", + "# sent_id = test-3", + "# text = im Haus", + line("1-2", "im", "_", "_", "_", "_", "_", "_", "_", "_"), + line("1", "in", "in", "ADP", "APPR", "_", "2", "case", "_", "_"), + line("2", "Haus", "Haus", "NOUN", "NN", "_", "0", "root", "_", "_"), + "", + "# sent_id = test-4", + "# text = Dogs bark", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "") + "\n"; + + private static ConlluDependencySampleStream stream() throws IOException { + return new ConlluDependencySampleStream(new ConlluStream( + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), + ConlluTagset.U); + } + + @Test + void testReadsSamplesAndSkipsUnusableSentences() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + assertArrayEquals(new String[] {"PRON", "VERB", "DET", "NOUN"}, first.getTags()); + assertEquals(1, first.getGraph().headOf(0)); + assertEquals(DependencyArc.ROOT_HEAD, first.getGraph().headOf(1)); + assertEquals(3, first.getGraph().headOf(2)); + assertEquals(1, first.getGraph().headOf(3)); + assertEquals("obj", first.getGraph().relationOf(3)); + + // the underscore-head sentence and the merged-contraction sentence are both skipped + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"Dogs", "bark"}, second.getTokens()); + assertEquals(1, second.getGraph().headOf(0)); + + assertNull(samples.read()); + } + } + + @Test + void testNullTagsetThrows() { + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(new ConlluStream( + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java new file mode 100644 index 0000000000..bd2d05f0ec --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + + +/** + * The static oracle for the arc-standard system: derives the transition sequence that + * reproduces a gold {@link DependencyGraph}. + * + *

An arc is only created once all dependents of the token being attached have been + * collected, which is the arc-standard correctness condition. The oracle is defined for + * projective trees only; a non-projective gold graph has no arc-standard derivation and + * is rejected.

+ * + * @since 3.0.0 + */ +public final class ArcStandardOracle { + + private ArcStandardOracle() { + // static oracle, not meant to be instantiated + } + + /** + * Derives the gold transition sequence for a graph. + * + * @param gold The gold dependency graph. Must not be {@code null} and must be + * projective. + * @return The transitions that rebuild {@code gold} from the start configuration, in + * order. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code gold} is {@code null} or not + * projective. + */ + public static List transitions(DependencyGraph gold) { + if (gold == null) { + throw new IllegalArgumentException("gold must not be null"); + } + final int n = gold.size(); + final int[] goldDependents = new int[n]; + for (int i = 0; i < n; i++) { + final int head = gold.headOf(i); + if (head >= 0) { + goldDependents[head]++; + } + } + + final ArcStandardState state = new ArcStandardState(n); + final List transitions = new ArrayList<>(2 * n); + while (!state.isTerminal()) { + final Transition next = nextTransition(gold, goldDependents, state); + if (next == null) { + throw new IllegalArgumentException( + "gold graph has no arc-standard derivation (non-projective): " + gold); + } + state.apply(next); + transitions.add(next); + } + return transitions; + } + + /** + * Picks the gold transition for the current configuration, or {@code null} when the + * configuration is stuck, which only happens for non-projective input. + */ + private static Transition nextTransition(DependencyGraph gold, int[] goldDependents, + ArcStandardState state) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + if (s1 >= 0 && gold.headOf(s1) == s0) { + final Transition leftArc = Transition.leftArc(gold.relationOf(s1)); + if (state.canApply(leftArc)) { + return leftArc; + } + } + if (s0 >= 0 && s1 != ArcStandardState.NONE && gold.headOf(s0) == s1 + && state.assignedDependents(s0) == goldDependents[s0]) { + final Transition rightArc = Transition.rightArc(gold.relationOf(s0)); + if (state.canApply(rightArc)) { + return rightArc; + } + } + if (state.canApply(Transition.SHIFT)) { + return Transition.SHIFT; + } + return null; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java new file mode 100644 index 0000000000..a261d95f3e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + + +/** + * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining + * tokens, and the arcs assigned so far. + * + *

The stack bottom holds the artificial root, exposed as {@link #ROOT}. Positions that + * do not exist, such as the second stack element in the initial configuration, are exposed + * as {@link #NONE}. A right arc from the artificial root is only applicable once the buffer + * is empty and the root is the only other stack element, which guarantees every completed + * parse has exactly one sentence root.

+ * + *

Instances are confined to a single parse and must not be shared between threads.

+ * + * @since 3.0.0 + */ +public final class ArcStandardState { + + /** The stack value representing the artificial root node. */ + public static final int ROOT = -1; + + /** The value returned for stack or buffer positions that do not exist. */ + public static final int NONE = -2; + + private final int tokenCount; + private final int[] stack; + private final int[] heads; + private final String[] relations; + private final int[] assignedDependents; + + private int top; + private int bufferFront; + + /** + * Initializes the start configuration for a sentence: the artificial root on the stack + * and every token in the buffer. + * + * @param tokenCount The number of tokens in the sentence. Must be greater than zero. + * @throws IllegalArgumentException Thrown if {@code tokenCount} is not positive. + */ + public ArcStandardState(int tokenCount) { + if (tokenCount <= 0) { + throw new IllegalArgumentException("tokenCount must be positive: " + tokenCount); + } + this.tokenCount = tokenCount; + this.stack = new int[tokenCount + 1]; + this.stack[0] = ROOT; + this.top = 0; + this.bufferFront = 0; + this.heads = new int[tokenCount]; + this.relations = new String[tokenCount]; + this.assignedDependents = new int[tokenCount]; + } + + /** + * @return {@code true} if the buffer is empty and only the artificial root remains on + * the stack, so the parse is complete. + */ + public boolean isTerminal() { + return bufferFront == tokenCount && top == 0; + } + + /** + * Checks whether a transition may be applied in the current configuration. + * + * @param transition The transition to check. Must not be {@code null}. + * @return {@code true} if {@link #apply(Transition)} would succeed. + * @throws IllegalArgumentException Thrown if {@code transition} is {@code null}. + */ + public boolean canApply(Transition transition) { + if (transition == null) { + throw new IllegalArgumentException("transition must not be null"); + } + return switch (transition.type()) { + case SHIFT -> bufferFront < tokenCount; + case LEFT_ARC -> top >= 2; + case RIGHT_ARC -> top >= 2 || (top == 1 && bufferFront == tokenCount); + }; + } + + /** + * Applies a transition, updating stack, buffer, and arcs. + * + * @param transition The transition to apply. Must not be {@code null} and must be + * applicable per {@link #canApply(Transition)}. + * @throws IllegalArgumentException Thrown if the transition is {@code null} or not + * applicable in the current configuration. + */ + public void apply(Transition transition) { + if (!canApply(transition)) { + throw new IllegalArgumentException("transition not applicable: " + transition + + " in " + this); + } + switch (transition.type()) { + case SHIFT -> { + top++; + stack[top] = bufferFront++; + } + case LEFT_ARC -> { + final int dependent = stack[top - 1]; + attach(stack[top], dependent, transition.label()); + stack[top - 1] = stack[top]; + top--; + } + case RIGHT_ARC -> { + attach(stack[top - 1], stack[top], transition.label()); + top--; + } + default -> throw new IllegalArgumentException("unsupported type: " + transition.type()); + } + } + + private void attach(int head, int dependent, String relation) { + heads[dependent] = head; + relations[dependent] = relation; + if (head >= 0) { + assignedDependents[head]++; + } + } + + /** + * Retrieves a stack element counted from the top. + * + * @param fromTop Zero for the top element, one for the element below it, and so on. + * Must not be negative. + * @return The token index at that position, {@link #ROOT} for the artificial root, or + * {@link #NONE} if the position does not exist. + * @throws IllegalArgumentException Thrown if {@code fromTop} is negative. + */ + public int stack(int fromTop) { + if (fromTop < 0) { + throw new IllegalArgumentException("fromTop must not be negative: " + fromTop); + } + final int position = top - fromTop; + return position < 0 ? NONE : stack[position]; + } + + /** + * Retrieves a buffer element counted from the front. + * + * @param fromFront Zero for the next token to be shifted, one for the token after it, + * and so on. Must not be negative. + * @return The token index at that position, or {@link #NONE} if the position does not + * exist. + * @throws IllegalArgumentException Thrown if {@code fromFront} is negative. + */ + public int buffer(int fromFront) { + if (fromFront < 0) { + throw new IllegalArgumentException("fromFront must not be negative: " + fromFront); + } + final int position = bufferFront + fromFront; + return position >= tokenCount ? NONE : position; + } + + /** + * @return The number of stack elements including the artificial root. + */ + public int stackSize() { + return top + 1; + } + + /** + * @return The number of tokens still in the buffer. + */ + public int bufferSize() { + return tokenCount - bufferFront; + } + + /** + * Retrieves how many dependents have been attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The number of arcs assigned with the token as head. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int assignedDependents(int index) { + if (index < 0 || index >= tokenCount) { + throw new IllegalArgumentException("token index out of range: " + index); + } + return assignedDependents[index]; + } + + /** + * Builds the {@link DependencyGraph} of a completed parse. + * + * @return The parsed graph. Never {@code null}. + * @throws IllegalStateException Thrown if the parse is not yet terminal. + */ + public DependencyGraph toGraph() { + if (!isTerminal()) { + throw new IllegalStateException("parse is not terminal: " + this); + } + return DependencyGraph.of(heads, relations); + } + + @Override + public String toString() { + return "stackSize=" + stackSize() + ", bufferSize=" + bufferSize() + + ", tokenCount=" + tokenCount; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java new file mode 100644 index 0000000000..90010cbf50 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * Generates the classification features for one arc-standard configuration: words and tags + * of the topmost stack and frontmost buffer positions, their pairings, and a bucketed + * distance between stack top and buffer front. + * + *

Instances hold no state and are safe to share between threads.

+ * + * @since 3.0.0 + */ +public class DependencyContextGenerator { + + private static final String ROOT_VALUE = "*ROOT*"; + private static final String NONE_VALUE = "*NULL*"; + + /** + * Generates the features of the current configuration. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The feature strings. Never {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public String[] getContext(ArcStandardState state, String[] tokens, String[] tags) { + if (state == null || tokens == null || tags == null) { + throw new IllegalArgumentException("state, tokens and tags must not be null"); + } + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int s2 = state.stack(2); + final int b0 = state.buffer(0); + final int b1 = state.buffer(1); + final int b2 = state.buffer(2); + + final String s0w = word(tokens, s0); + final String s0t = tag(tags, s0); + final String s1w = word(tokens, s1); + final String s1t = tag(tags, s1); + final String s2t = tag(tags, s2); + final String b0w = word(tokens, b0); + final String b0t = tag(tags, b0); + final String b1t = tag(tags, b1); + final String b2t = tag(tags, b2); + + final List features = new ArrayList<>(20); + features.add("s0w=" + s0w); + features.add("s0t=" + s0t); + features.add("s1w=" + s1w); + features.add("s1t=" + s1t); + features.add("s2t=" + s2t); + features.add("b0w=" + b0w); + features.add("b0t=" + b0t); + features.add("b1t=" + b1t); + features.add("b2t=" + b2t); + features.add("s0wt=" + s0w + '/' + s0t); + features.add("s0w,b0w=" + s0w + '|' + b0w); + features.add("s0t,b0t=" + s0t + '|' + b0t); + features.add("s1t,s0t=" + s1t + '|' + s0t); + features.add("s1t,s0t,b0t=" + s1t + '|' + s0t + '|' + b0t); + features.add("s0t,b0t,b1t=" + s0t + '|' + b0t + '|' + b1t); + features.add("s0deps=" + dependents(state, s0)); + features.add("s1deps=" + dependents(state, s1)); + features.add("dist=" + distance(s0, b0)); + return features.toArray(new String[0]); + } + + private static String word(String[] tokens, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; + } + + private static String tag(String[] tags, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tags[index]; + } + + private static String dependents(ArcStandardState state, int index) { + return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), 3)); + } + + private static String distance(int s0, int b0) { + if (s0 < 0 || b0 < 0) { + return NONE_VALUE; + } + final int distance = b0 - s0; + return distance >= 4 ? "4+" : Integer.toString(distance); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java new file mode 100644 index 0000000000..f29bada722 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.eval.Evaluator; +import opennlp.tools.util.eval.Mean; + +/** + * Measures the quality of a {@link DependencyParser} against gold + * {@link DependencySample samples} with the two standard scores: the unlabeled attachment + * score (UAS, the fraction of tokens with the correct head) and the labeled attachment + * score (LAS, the fraction of tokens with the correct head and relation label). + * + * @since 3.0.0 + */ +public class DependencyEvaluator extends Evaluator { + + private final DependencyParser parser; + private final Mean uas = new Mean(); + private final Mean las = new Mean(); + + /** + * Initializes a {@link DependencyEvaluator}. + * + * @param parser The parser to evaluate. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyEvaluator(DependencyParser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + /** + * Parses the sample's sentence and scores the prediction against the gold graph. + * + * @param reference The gold sample. Must not be {@code null}. + * @return A {@link DependencySample} carrying the predicted graph. Never {@code null}. + */ + @Override + protected DependencySample processSample(DependencySample reference) { + final DependencyGraph gold = reference.getGraph(); + final DependencyGraph predicted = parser.parse(reference.getTokens(), reference.getTags()); + for (int i = 0; i < gold.size(); i++) { + final boolean headMatches = gold.headOf(i) == predicted.headOf(i); + uas.add(headMatches ? 1 : 0); + las.add(headMatches && gold.relationOf(i).equals(predicted.relationOf(i)) ? 1 : 0); + } + return new DependencySample(reference.getTokens(), reference.getTags(), predicted); + } + + /** + * @return The unlabeled attachment score over all evaluated tokens. + */ + public double getUas() { + return uas.mean(); + } + + /** + * @return The labeled attachment score over all evaluated tokens. + */ + public double getLas() { + return las.mean(); + } + + /** + * @return The number of tokens scored so far. + */ + public long getWordCount() { + return uas.count(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java new file mode 100644 index 0000000000..87fdf27300 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.ml.model.Event; +import opennlp.tools.util.ObjectStream; + +/** + * Turns {@link DependencySample samples} into training {@link Event events}: for each + * sample the {@link ArcStandardOracle} derives the gold transitions, and every transition + * becomes one event pairing the configuration features with the encoded transition. + * + *

Samples whose graph has no arc-standard derivation, that is non-projective trees, + * are skipped and counted; the count is logged once the stream is exhausted.

+ */ +class DependencyEventStream implements ObjectStream { + + private static final Logger logger = LoggerFactory.getLogger(DependencyEventStream.class); + + private final ObjectStream samples; + private final DependencyContextGenerator contextGenerator; + private final Deque pending = new ArrayDeque<>(); + + private int skipped; + + /** + * Initializes the stream. + * + * @param samples The samples to convert. Must not be {@code null}. + * @param contextGenerator The feature generator. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + DependencyEventStream(ObjectStream samples, + DependencyContextGenerator contextGenerator) { + if (samples == null || contextGenerator == null) { + throw new IllegalArgumentException("samples and contextGenerator must not be null"); + } + this.samples = samples; + this.contextGenerator = contextGenerator; + } + + @Override + public Event read() throws IOException { + while (pending.isEmpty()) { + final DependencySample sample = samples.read(); + if (sample == null) { + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + skipped = 0; + } + return null; + } + final List transitions; + try { + transitions = ArcStandardOracle.transitions(sample.getGraph()); + } catch (IllegalArgumentException e) { + skipped++; + continue; + } + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : transitions) { + pending.add(new Event(transition.encode(), + contextGenerator.getContext(state, tokens, tags))); + state.apply(transition); + } + } + return pending.poll(); + } + + @Override + public void reset() throws IOException, UnsupportedOperationException { + samples.reset(); + pending.clear(); + skipped = 0; + } + + @Override + public void close() throws IOException { + samples.close(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java new file mode 100644 index 0000000000..76d8814910 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.HashMap; + +import opennlp.tools.ml.EventTrainer; +import opennlp.tools.ml.TrainerFactory; +import opennlp.tools.ml.TrainerFactory.TrainerType; +import opennlp.tools.ml.model.Event; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.TrainingParameters; + +/** + * A greedy transition-based {@link DependencyParser}: a maximum entropy classifier picks + * the next arc-standard {@link Transition} for each configuration until the parse is + * complete, always taking the highest scoring transition that is applicable. + * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see DependencyParser + * @since 3.0.0 + */ +public class DependencyParserME implements DependencyParser { + + private final MaxentModel model; + private final DependencyContextGenerator contextGenerator; + + /** + * Initializes a {@link DependencyParserME} with a trained transition model. + * + * @param model The transition classification model. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + */ + public DependencyParserME(MaxentModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model; + this.contextGenerator = new DependencyContextGenerator(); + } + + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + state.apply(bestApplicable(state, tokens, tags)); + } + return state.toGraph(); + } + + /** + * Scores all outcomes for the current configuration and picks the best transition that + * is applicable; inapplicable outcomes are passed over regardless of score. + */ + private Transition bestApplicable(ArcStandardState state, String[] tokens, String[] tags) { + final double[] probabilities = model.eval(contextGenerator.getContext(state, tokens, tags)); + Transition best = null; + double bestProbability = Double.NEGATIVE_INFINITY; + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] <= bestProbability) { + continue; + } + final Transition candidate; + try { + candidate = Transition.decode(model.getOutcome(i)); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "model outcome is not a transition: " + model.getOutcome(i), e); + } + if (state.canApply(candidate)) { + best = candidate; + bestProbability = probabilities[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + return best; + } + + /** + * Trains a greedy arc-standard parser from dependency samples. + * + *

Non-projective samples have no arc-standard derivation and are skipped during + * event generation.

+ * + * @param samples The training samples. Must not be {@code null}. + * @param parameters The {@link TrainingParameters}. Must not be {@code null} and must + * select an event model trainer. + * @return A trained {@link DependencyParserME}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or the + * configured trainer is not an event model trainer. + */ + public static DependencyParserME train(ObjectStream samples, + TrainingParameters parameters) throws IOException { + if (samples == null || parameters == null) { + throw new IllegalArgumentException("samples and parameters must not be null"); + } + final TrainerType trainerType = TrainerFactory.getTrainerType(parameters); + if (!TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { + throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); + } + final EventTrainer trainer = + TrainerFactory.getEventTrainer(parameters, new HashMap<>()); + final ObjectStream events = + new DependencyEventStream(samples, new DependencyContextGenerator()); + return new DependencyParserME(trainer.train(events)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java new file mode 100644 index 0000000000..0f9977e675 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * One action of the arc-standard transition system: shift the next buffer token onto the + * stack, or attach one of the two topmost stack tokens to the other under a relation label. + * + *

A transition doubles as a classification outcome: {@link #encode()} renders it as the + * outcome string a model is trained on, and {@link #decode(String)} restores it.

+ * + * @param type The kind of action. Must not be {@code null}. + * @param label The relation label for arc actions, {@code null} for {@link Type#SHIFT}. + * + * @since 3.0.0 + */ +public record Transition(Type type, String label) { + + /** + * The kinds of arc-standard action. + */ + public enum Type { + /** Pushes the front of the buffer onto the stack. */ + SHIFT, + /** Attaches the second stack token to the top one and removes the second. */ + LEFT_ARC, + /** Attaches the top stack token to the second one and removes the top. */ + RIGHT_ARC + } + + /** The single shift transition; shifts carry no label. */ + public static final Transition SHIFT = new Transition(Type.SHIFT, null); + + private static final char SEPARATOR = ':'; + + /** + * Validates the pairing of type and label. + * + * @throws IllegalArgumentException Thrown if {@code type} is {@code null}, a shift + * carries a label, or an arc action has a {@code null} or blank label. + */ + public Transition { + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + if (type == Type.SHIFT) { + if (label != null) { + throw new IllegalArgumentException("a shift must not carry a label: " + label); + } + } else if (label == null || label.isBlank()) { + throw new IllegalArgumentException("an arc transition needs a relation label"); + } + } + + /** + * Creates a left-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#LEFT_ARC}. Never {@code null}. + */ + public static Transition leftArc(String label) { + return new Transition(Type.LEFT_ARC, label); + } + + /** + * Creates a right-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#RIGHT_ARC}. Never {@code null}. + */ + public static Transition rightArc(String label) { + return new Transition(Type.RIGHT_ARC, label); + } + + /** + * Renders the transition as a model outcome string, for example {@code SHIFT} or + * {@code LEFT_ARC:nsubj}. + * + * @return The outcome string. Never {@code null}. + */ + public String encode() { + return type == Type.SHIFT ? type.name() : type.name() + SEPARATOR + label; + } + + /** + * Restores a transition from a model outcome string produced by {@link #encode()}. + * + * @param outcome The outcome string. Must not be {@code null}. + * @return The decoded {@link Transition}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code outcome} is {@code null} or does not + * name a valid transition. + */ + public static Transition decode(String outcome) { + if (outcome == null) { + throw new IllegalArgumentException("outcome must not be null"); + } + if (Type.SHIFT.name().equals(outcome)) { + return SHIFT; + } + final int separator = outcome.indexOf(SEPARATOR); + if (separator < 0) { + throw new IllegalArgumentException("not a transition outcome: " + outcome); + } + final Type type; + try { + type = Type.valueOf(outcome.substring(0, separator)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("not a transition outcome: " + outcome, e); + } + return new Transition(type, outcome.substring(separator + 1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java new file mode 100644 index 0000000000..6cfa9f57ca --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link ArcStandardOracle} derivations replay to the gold graph through + * {@link ArcStandardState}, and that non-projective input is rejected. + */ +public class ArcStandardOracleTest { + + private static DependencyGraph replay(DependencyGraph gold) { + final List transitions = ArcStandardOracle.transitions(gold); + // every token is shifted once and attached once + assertEquals(2 * gold.size(), transitions.size()); + final ArcStandardState state = new ArcStandardState(gold.size()); + for (final Transition transition : transitions) { + assertTrue(state.canApply(transition)); + state.apply(transition); + } + return state.toGraph(); + } + + @Test + void testRoundTripSimpleSentence() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripSingleToken() { + final DependencyGraph gold = DependencyGraph.of(new int[] {-1}, new String[] {"root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripRightBranching() { + // "eat fresh fish now": root with a right dependent that has its own left dependent + final DependencyGraph gold = DependencyGraph.of(new int[] {-1, 2, 0, 0}, + new String[] {"root", "amod", "obj", "advmod"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripDeepChain() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"a", "b", "c", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testNonProjectiveThrows() { + // arcs (2,0) and (3,1) cross, so there is no arc-standard derivation + final DependencyGraph nonProjective = DependencyGraph.of(new int[] {2, 3, -1, 2}, + new String[] {"a", "b", "root", "c"}); + assertThrows(IllegalArgumentException.class, + () -> ArcStandardOracle.transitions(nonProjective)); + } + + @Test + void testNullGraphThrows() { + assertThrows(IllegalArgumentException.class, () -> ArcStandardOracle.transitions(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java new file mode 100644 index 0000000000..def76900fe --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyParserME} end to end: training on a tiny corpus must let the + * greedy parser reproduce the training sentences, which proves the oracle, event stream, + * feature generation, and decode loop agree with each other. + */ +public class DependencyParserMETest { + + private static DependencyParserME parser; + + private static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + private static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + parser = DependencyParserME.train(ObjectStreamUtils.createObjectStream(corpus()), + parameters); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testParseAlwaysYieldsASingleRootedTree() { + // an unseen sentence must still decode to a valid graph, whatever its quality + final DependencyGraph parsed = parser.parse(new String[] {"cats", "sleep"}, + new String[] {"NNS", "VBP"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(320, evaluator.getWordCount()); + } + + @Test + void testParseValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } + + @Test + void testConstructorRejectsNullModel() { + assertThrows(IllegalArgumentException.class, () -> new DependencyParserME(null)); + } + + @Test + void testTrainValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(null, TrainingParameters.defaultParams())); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train( + ObjectStreamUtils.createObjectStream(corpus()), null)); + } +} From f5c11dc46d8e9b4c7da741fb896fb8a20c09b52f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 05:23:30 -0400 Subject: [PATCH 63/92] depparse: DependencyModel persistence and a gated Universal Dependencies evaluation Adds DependencyModel on the standard BaseModel machinery, so trained parsers serialize and load like every other tool model; the declared serialVersionUID is the serialver-computed value. Training now returns the model with the trainer's manifest entries, and DependencyParserME gains a model constructor beside the raw one. The round-trip test serializes, reloads, and re-parses. ConlluDependencyParserEvalTest trains on a Universal Dependencies treebank and scores UAS and LAS on its test split; it runs only when opennlp.depparse.ud.dir names the downloaded splits, keeps treebank data out of the repository, and asserts only a low regression floor, with the logged scores as the measurement. (cherry picked from commit c5a3b946b6d8a06a01ff07b961a74ab26a671f3f) --- .../ConlluDependencyParserEvalTest.java | 84 +++++++++++++ .../tools/depparse/DependencyModel.java | 110 ++++++++++++++++++ .../tools/depparse/DependencyParserME.java | 38 ++++-- .../depparse/DependencyParserMETest.java | 37 +++++- 4 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java new file mode 100644 index 0000000000..558547cac8 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.MarkableFileInputStreamFactory; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Trains the greedy arc-standard parser on a Universal Dependencies treebank and scores + * it on the treebank's test split, reporting UAS and LAS. + * + *

Runs only when {@code opennlp.depparse.ud.dir} names a directory containing + * {@code train.conllu} and {@code test.conllu} (a UD treebank's splits, renamed or + * linked). The data is downloaded by the runner and never enters the repository; check + * the treebank's own license before training models for distribution. The assertion is + * a low regression floor; the logged scores are the measurement.

+ */ +public class ConlluDependencyParserEvalTest { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencyParserEvalTest.class); + + @Test + @EnabledIfSystemProperty(named = "opennlp.depparse.ud.dir", matches = ".+") + void testTrainAndScoreOnUniversalDependencies() throws IOException { + final Path dir = Path.of(System.getProperty("opennlp.depparse.ud.dir")); + + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 5); + final long trainStart = System.currentTimeMillis(); + final DependencyModel model; + try (ConlluDependencySampleStream train = samples(dir.resolve("train.conllu"))) { + model = DependencyParserME.train("eng", train, parameters); + } + logger.info("trained in {} ms", System.currentTimeMillis() - trainStart); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new DependencyParserME(model)); + try (ConlluDependencySampleStream test = samples(dir.resolve("test.conllu"))) { + evaluator.evaluate(test); + } + logger.info("UAS {} LAS {} over {} tokens", + evaluator.getUas(), evaluator.getLas(), evaluator.getWordCount()); + + // a regression floor, far below any plausible result; the log line is the measurement + assertTrue(evaluator.getUas() > 0.6d, "UAS regressed below the floor"); + assertTrue(evaluator.getLas() > 0.5d, "LAS regressed below the floor"); + } + + private static ConlluDependencySampleStream samples(Path conllu) throws IOException { + final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); + return new ConlluDependencySampleStream(new ConlluStream(in), ConlluTagset.U); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java new file mode 100644 index 0000000000..8f2c608eeb --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; + +import opennlp.tools.ml.model.AbstractModel; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.model.BaseModel; + +/** + * The persisted form of a trained {@link DependencyParserME}: the transition + * classification model plus the standard model manifest, serialized and loaded through + * the same {@link BaseModel} machinery as every other tool model. + * + * @see DependencyParserME + * @since 3.0.0 + */ +public class DependencyModel extends BaseModel { + + private static final long serialVersionUID = -2928968185269611443L; + + private static final String COMPONENT_NAME = "DependencyParserME"; + static final String PARSER_MODEL_ENTRY_NAME = "depparse.model"; + + /** + * Initializes a {@link DependencyModel} from a trained transition model. + * + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. + * @param parserModel The transition classification model. Must not be {@code null}. + * @param manifestInfoEntries Additional entries for the manifest, or {@code null}. + * @throws IllegalArgumentException Thrown if {@code parserModel} is {@code null}. + */ + public DependencyModel(String languageCode, MaxentModel parserModel, + Map manifestInfoEntries) { + super(COMPONENT_NAME, languageCode, manifestInfoEntries); + if (parserModel == null) { + throw new IllegalArgumentException("parserModel must not be null"); + } + artifactMap.put(PARSER_MODEL_ENTRY_NAME, parserModel); + checkArtifactMap(); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model. + * + * @param in The stream to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(InputStream in) throws IOException { + super(COMPONENT_NAME, in); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelFile The file to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(File modelFile) throws IOException { + super(COMPONENT_NAME, modelFile); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelPath The path to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + */ + public DependencyModel(Path modelPath) throws IOException { + super(COMPONENT_NAME, modelPath); + } + + @Override + protected void validateArtifactMap() throws InvalidFormatException { + super.validateArtifactMap(); + if (!(artifactMap.get(PARSER_MODEL_ENTRY_NAME) instanceof AbstractModel)) { + throw new InvalidFormatException("The " + PARSER_MODEL_ENTRY_NAME + + " artifact is missing or not a supported transition model."); + } + } + + /** + * @return The transition classification model. Never {@code null}. + */ + public MaxentModel getParserModel() { + return (MaxentModel) artifactMap.get(PARSER_MODEL_ENTRY_NAME); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java index 76d8814910..bbd2d219ff 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Map; import opennlp.tools.ml.EventTrainer; import opennlp.tools.ml.TrainerFactory; @@ -45,7 +46,21 @@ public class DependencyParserME implements DependencyParser { private final DependencyContextGenerator contextGenerator; /** - * Initializes a {@link DependencyParserME} with a trained transition model. + * Initializes a {@link DependencyParserME} from a {@link DependencyModel}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + */ + public DependencyParserME(DependencyModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model.getParserModel(); + this.contextGenerator = new DependencyContextGenerator(); + } + + /** + * Initializes a {@link DependencyParserME} with a raw transition model. * * @param model The transition classification model. Must not be {@code null}. * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. @@ -109,32 +124,37 @@ private Transition bestApplicable(ArcStandardState state, String[] tokens, Strin } /** - * Trains a greedy arc-standard parser from dependency samples. + * Trains a greedy arc-standard parser model from dependency samples. * *

Non-projective samples have no arc-standard derivation and are skipped during * event generation.

* + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. * @param samples The training samples. Must not be {@code null}. * @param parameters The {@link TrainingParameters}. Must not be {@code null} and must * select an event model trainer. - * @return A trained {@link DependencyParserME}. Never {@code null}. + * @return A trained {@link DependencyModel}. Never {@code null}. * @throws IOException Thrown if reading the samples fails. * @throws IllegalArgumentException Thrown if a parameter is {@code null} or the * configured trainer is not an event model trainer. */ - public static DependencyParserME train(ObjectStream samples, - TrainingParameters parameters) throws IOException { - if (samples == null || parameters == null) { - throw new IllegalArgumentException("samples and parameters must not be null"); + public static DependencyModel train(String languageCode, + ObjectStream samples, TrainingParameters parameters) + throws IOException { + if (languageCode == null || samples == null || parameters == null) { + throw new IllegalArgumentException( + "languageCode, samples and parameters must not be null"); } final TrainerType trainerType = TrainerFactory.getTrainerType(parameters); if (!TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } + final Map manifestInfoEntries = new HashMap<>(); final EventTrainer trainer = - TrainerFactory.getEventTrainer(parameters, new HashMap<>()); + TrainerFactory.getEventTrainer(parameters, manifestInfoEntries); final ObjectStream events = new DependencyEventStream(samples, new DependencyContextGenerator()); - return new DependencyParserME(trainer.train(events)); + return new DependencyModel(languageCode, trainer.train(events), manifestInfoEntries); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java index def76900fe..62550ad239 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -38,6 +38,7 @@ */ public class DependencyParserMETest { + private static DependencyModel model; private static DependencyParserME parser; private static DependencySample sample(String[] tokens, String[] tags, int[] heads, @@ -64,8 +65,9 @@ private static List corpus() { static void trainParser() throws IOException { final TrainingParameters parameters = TrainingParameters.defaultParams(); parameters.put(Parameters.CUTOFF_PARAM, 0); - parser = DependencyParserME.train(ObjectStreamUtils.createObjectStream(corpus()), - parameters); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters); + parser = new DependencyParserME(model); } @Test @@ -108,15 +110,40 @@ void testParseValidatesArguments() { @Test void testConstructorRejectsNullModel() { - assertThrows(IllegalArgumentException.class, () -> new DependencyParserME(null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((DependencyModel) null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((opennlp.tools.ml.model.MaxentModel) null)); } @Test void testTrainValidatesArguments() { assertThrows(IllegalArgumentException.class, - () -> DependencyParserME.train(null, TrainingParameters.defaultParams())); + () -> DependencyParserME.train("eng", null, TrainingParameters.defaultParams())); assertThrows(IllegalArgumentException.class, - () -> DependencyParserME.train( + () -> DependencyParserME.train("eng", ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(null, + ObjectStreamUtils.createObjectStream(corpus()), + TrainingParameters.defaultParams())); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + model.serialize(out); + final DependencyModel reloaded = new DependencyModel( + new java.io.ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new DependencyParserME(reloaded) + .parse(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testModelRejectsNullParserModel() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel("eng", null, null)); } } From 1d8c366e28b5568740c70029faf2c4a27d016b3c Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 05:39:22 -0400 Subject: [PATCH 64/92] depparse: Read CoNLL-U raw so multiword-token sentences keep their dependencies Rewrites ConlluDependencySampleStream to parse CoNLL-U content directly instead of consuming the merged ConlluSentence view: ConlluStream merges multiword token ranges with their syntactic words, which suits the token and lemma views but destroys the dependency annotation of every contraction-bearing sentence. The raw reader drops range lines and empty nodes while keeping the syntactic words, so those sentences train and evaluate normally; on the English EWT treebank this recovers 2192 training and 302 test sentences that were previously skipped, and evaluation now covers the full test set. (cherry picked from commit f22b0acb107e52b68b87e56a2d24863df159dd5b) --- .../conllu/ConlluDependencySampleStream.java | 130 ++++++++++++------ .../ConlluDependencyParserEvalTest.java | 2 +- .../ConlluDependencySampleStreamTest.java | 63 +++++++-- 3 files changed, 140 insertions(+), 55 deletions(-) diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 0dae3f7fc6..22f9de9018 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -17,7 +17,10 @@ package opennlp.tools.formats.conllu; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -26,55 +29,68 @@ import opennlp.tools.depparse.DependencyGraph; import opennlp.tools.depparse.DependencySample; -import opennlp.tools.util.FilterObjectStream; +import opennlp.tools.util.InputStreamFactory; import opennlp.tools.util.ObjectStream; /** - * Reads {@link DependencySample samples} from a stream of {@link ConlluSentence sentences}, - * mapping the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * Reads {@link DependencySample samples} directly from CoNLL-U content, mapping the + * {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. * - *

Empty nodes carry no basic dependency annotation and are dropped; the remaining word - * lines keep their one-based ids, which are shifted to the zero-based indices of - * {@link DependencyGraph}. Sentences containing a multiword token are skipped entirely: - * {@link ConlluStream} merges such a range with its syntactic words, so the dependency - * annotation of those words is no longer recoverable. Sentences whose annotation is - * incomplete or invalid, for example an underscore head, are skipped as well. Skips are - * counted and the count is logged once the stream is exhausted.

+ *

The file is parsed raw, deliberately not through {@link ConlluStream}: that stream + * merges multiword token ranges with their syntactic words, which suits the token and + * lemma views but destroys the dependency annotation of every sentence containing a + * contraction. Here range lines and empty nodes are dropped while their syntactic words + * are kept, so contraction-bearing sentences train and evaluate normally. Sentences + * whose annotation is incomplete or invalid, for example an underscore head, are + * skipped and counted; the count is logged once the stream is exhausted.

* * @since 3.0.0 */ -public class ConlluDependencySampleStream - extends FilterObjectStream { +public class ConlluDependencySampleStream implements ObjectStream { private static final Logger logger = LoggerFactory.getLogger(ConlluDependencySampleStream.class); - private final ConlluTagset tagset; + private static final int COLUMNS = 10; + private static final int FORM = 1; + private static final int UPOS = 3; + private static final int XPOS = 4; + private static final int HEAD = 6; + private static final int DEPREL = 7; + private final InputStreamFactory in; + private final int tagColumn; + + private BufferedReader reader; private int skipped; /** * Initializes the stream. * - * @param samples The sentences to convert. Must not be {@code null}. - * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must not - * be {@code null}. + * @param in The CoNLL-U content. Must not be {@code null}. + * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must + * not be {@code null}. + * @throws IOException Thrown if opening the content fails. * @throws IllegalArgumentException Thrown if any parameter is {@code null}. */ - public ConlluDependencySampleStream(ObjectStream samples, - ConlluTagset tagset) { - super(samples); + public ConlluDependencySampleStream(InputStreamFactory in, ConlluTagset tagset) + throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } if (tagset == null) { throw new IllegalArgumentException("tagset must not be null"); } - this.tagset = tagset; + this.in = in; + this.tagColumn = tagset == ConlluTagset.U ? UPOS : XPOS; + this.reader = open(); } @Override public DependencySample read() throws IOException { - ConlluSentence sentence; - while ((sentence = samples.read()) != null) { - final DependencySample sample = convert(sentence); + List words; + while (!(words = nextSentence()).isEmpty()) { + final DependencySample sample = convert(words); if (sample != null) { return sample; } @@ -89,35 +105,50 @@ public DependencySample read() throws IOException { } /** - * Converts one sentence, or returns {@code null} when its annotation is unusable. + * Reads the syntactic word lines of the next sentence: comments, multiword token + * ranges, and empty nodes are dropped; an empty list means the end of the content. */ - private DependencySample convert(ConlluSentence sentence) { - final List words = new ArrayList<>(); - for (final ConlluWordLine line : sentence.getWordLines()) { - final String id = line.getId(); - if (id.indexOf('-') >= 0) { - // a merged multiword token: its syntactic words are gone, the sentence is unusable - return null; + private List nextSentence() throws IOException { + final List words = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + if (line.isBlank()) { + if (!words.isEmpty()) { + return words; + } + continue; } - if (id.indexOf('.') < 0) { - words.add(line); + if (line.charAt(0) == '#') { + continue; + } + final String[] fields = line.split("\t", -1); + if (fields.length < COLUMNS) { + throw new IOException("not a CoNLL-U word line: " + line); + } + final String id = fields[0]; + if (id.indexOf('-') < 0 && id.indexOf('.') < 0) { + words.add(fields); } } - if (words.isEmpty()) { - return null; - } + return words; + } + + /** + * Converts one sentence, or returns {@code null} when its annotation is unusable. + */ + private DependencySample convert(List words) { final int n = words.size(); final String[] tokens = new String[n]; final String[] tags = new String[n]; final int[] heads = new int[n]; final String[] relations = new String[n]; for (int i = 0; i < n; i++) { - final ConlluWordLine word = words.get(i); - tokens[i] = word.getForm(); - tags[i] = word.getPosTag(tagset); - relations[i] = word.getDeprel(); + final String[] word = words.get(i); + tokens[i] = word[FORM]; + tags[i] = word[tagColumn]; + relations[i] = word[DEPREL]; try { - heads[i] = Integer.parseInt(word.getHead()) - 1; + heads[i] = Integer.parseInt(word[HEAD]) - 1; } catch (NumberFormatException e) { return null; } @@ -128,4 +159,21 @@ private DependencySample convert(ConlluSentence sentence) { return null; } } + + @Override + public void reset() throws IOException, UnsupportedOperationException { + reader.close(); + reader = open(); + skipped = 0; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private BufferedReader open() throws IOException { + return new BufferedReader( + new InputStreamReader(in.createInputStream(), StandardCharsets.UTF_8)); + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java index 558547cac8..af4cc1e919 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -79,6 +79,6 @@ void testTrainAndScoreOnUniversalDependencies() throws IOException { private static ConlluDependencySampleStream samples(Path conllu) throws IOException { final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); - return new ConlluDependencySampleStream(new ConlluStream(in), ConlluTagset.U); + return new ConlluDependencySampleStream(in, ConlluTagset.U); } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index 6105efd00e..b7fb44cedd 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -25,6 +25,7 @@ import opennlp.tools.depparse.DependencyArc; import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,9 +34,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; /** - * Tests that {@link ConlluDependencySampleStream} maps the basic dependency columns and - * skips sentences without a usable annotation, including sentences whose multiword tokens - * were merged by {@link ConlluStream}. + * Tests that the raw reader maps the basic dependency columns, keeps the syntactic + * words of multiword tokens while dropping the range line itself, and skips sentences + * without a usable annotation. */ public class ConlluDependencySampleStreamTest { @@ -67,14 +68,16 @@ private static String line(String... fields) { line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), "") + "\n"; + private static InputStreamFactory factory() { + return () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + } + private static ConlluDependencySampleStream stream() throws IOException { - return new ConlluDependencySampleStream(new ConlluStream( - () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), - ConlluTagset.U); + return new ConlluDependencySampleStream(factory(), ConlluTagset.U); } @Test - void testReadsSamplesAndSkipsUnusableSentences() throws IOException { + void testReadsSamplesKeepsContractionsAndSkipsUnusableSentences() throws IOException { try (ConlluDependencySampleStream samples = stream()) { final DependencySample first = samples.read(); assertNotNull(first); @@ -83,23 +86,57 @@ void testReadsSamplesAndSkipsUnusableSentences() throws IOException { assertEquals(1, first.getGraph().headOf(0)); assertEquals(DependencyArc.ROOT_HEAD, first.getGraph().headOf(1)); assertEquals(3, first.getGraph().headOf(2)); - assertEquals(1, first.getGraph().headOf(3)); assertEquals("obj", first.getGraph().relationOf(3)); - // the underscore-head sentence and the merged-contraction sentence are both skipped + // the underscore-head sentence is skipped; the contraction sentence is KEPT, + // with the range line dropped and its syntactic words intact final DependencySample second = samples.read(); assertNotNull(second); - assertArrayEquals(new String[] {"Dogs", "bark"}, second.getTokens()); + assertArrayEquals(new String[] {"in", "Haus"}, second.getTokens()); assertEquals(1, second.getGraph().headOf(0)); + assertEquals("case", second.getGraph().relationOf(0)); + + final DependencySample third = samples.read(); + assertNotNull(third); + assertArrayEquals(new String[] {"Dogs", "bark"}, third.getTokens()); assertNull(samples.read()); } } @Test - void testNullTagsetThrows() { + void testResetRestartsTheStream() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + assertNotNull(samples.read()); + samples.reset(); + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + } + } + + @Test + void testXposTagsetSelectsTheOtherColumn() throws IOException { + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(factory(), ConlluTagset.X)) { + assertArrayEquals(new String[] {"PRP", "VBD", "DT", "NNS"}, + samples.read().getTags()); + } + } + + @Test + void testMalformedLineFailsLoud() { + final InputStreamFactory bad = () -> new ByteArrayInputStream( + "1\ttoo\tfew\tcolumns\n".getBytes(StandardCharsets.UTF_8)); + assertThrows(IOException.class, + () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); + } + + @Test + void testValidation() { + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(null, ConlluTagset.U)); assertThrows(IllegalArgumentException.class, - () -> new ConlluDependencySampleStream(new ConlluStream( - () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8))), null)); + () -> new ConlluDependencySampleStream(factory(), null)); } } From dc070db0e7e2420a63f6ca4c8e2776504205db4a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 05:39:22 -0400 Subject: [PATCH 65/92] depparse: Valency and child features for the transition classifier Extends the configuration features with the partial structure built so far: tags and relations of the leftmost and rightmost dependents of the top two stack tokens, their conjunctions with the neighboring tags, word and tag pairs in both directions, a second-order stack tag triple, buffer word context, and a distance-tag conjunction. ArcStandardState now tracks leftmost and rightmost dependents and exposes the assigned relation of an attached token. Measured together with the raw CoNLL-U reader on UD English EWT with gold UPOS tags: UAS 0.8259 and LAS 0.7929 over the full 25094-token test set, up from 0.7791 and 0.7103 over the 19394-token subset the merged reader could parse; training takes 91 seconds. (cherry picked from commit 6cf87b84bef8f253a57e1cdc9431f46f67642c04) --- .../tools/depparse/ArcStandardState.java | 54 +++++++++++++++- .../depparse/DependencyContextGenerator.java | 63 +++++++++++++++++-- 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index a261d95f3e..0464a3436d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -45,6 +45,8 @@ public final class ArcStandardState { private final int[] heads; private final String[] relations; private final int[] assignedDependents; + private final int[] leftmostDependents; + private final int[] rightmostDependents; private int top; private int bufferFront; @@ -68,6 +70,10 @@ public ArcStandardState(int tokenCount) { this.heads = new int[tokenCount]; this.relations = new String[tokenCount]; this.assignedDependents = new int[tokenCount]; + this.leftmostDependents = new int[tokenCount]; + this.rightmostDependents = new int[tokenCount]; + java.util.Arrays.fill(this.leftmostDependents, NONE); + java.util.Arrays.fill(this.rightmostDependents, NONE); } /** @@ -133,6 +139,12 @@ private void attach(int head, int dependent, String relation) { relations[dependent] = relation; if (head >= 0) { assignedDependents[head]++; + if (leftmostDependents[head] == NONE || dependent < leftmostDependents[head]) { + leftmostDependents[head] = dependent; + } + if (rightmostDependents[head] == NONE || dependent > rightmostDependents[head]) { + rightmostDependents[head] = dependent; + } } } @@ -192,10 +204,50 @@ public int bufferSize() { * @throws IllegalArgumentException Thrown if {@code index} is out of range. */ public int assignedDependents(int index) { + checkTokenIndex(index); + return assignedDependents[index]; + } + + private void checkTokenIndex(int index) { if (index < 0 || index >= tokenCount) { throw new IllegalArgumentException("token index out of range: " + index); } - return assignedDependents[index]; + } + + /** + * Retrieves the leftmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int leftmostDependent(int index) { + checkTokenIndex(index); + return leftmostDependents[index]; + } + + /** + * Retrieves the rightmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int rightmostDependent(int index) { + checkTokenIndex(index); + return rightmostDependents[index]; + } + + /** + * Retrieves the relation a token was attached under, when it has been attached. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The relation label, or {@code null} when the token is still unattached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String assignedRelation(int index) { + checkTokenIndex(index); + return relations[index]; } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java index 90010cbf50..4639bfcbb5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -21,9 +21,10 @@ import java.util.List; /** - * Generates the classification features for one arc-standard configuration: words and tags - * of the topmost stack and frontmost buffer positions, their pairings, and a bucketed - * distance between stack top and buffer front. + * Generates the classification features for one arc-standard configuration: words and + * tags of the topmost stack and frontmost buffer positions, their pairings, the partial + * structure built so far (tags and relations of the leftmost and rightmost dependents, + * valency counts), and a bucketed distance between stack top and buffer front. * *

Instances hold no state and are safe to share between threads.

* @@ -62,10 +63,19 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag final String s2t = tag(tags, s2); final String b0w = word(tokens, b0); final String b0t = tag(tags, b0); + final String b1w = word(tokens, b1); final String b1t = tag(tags, b1); final String b2t = tag(tags, b2); - final List features = new ArrayList<>(20); + final String s0lct = dependentTag(state, tags, s0, true); + final String s0rct = dependentTag(state, tags, s0, false); + final String s1lct = dependentTag(state, tags, s1, true); + final String s1rct = dependentTag(state, tags, s1, false); + final String s0lcl = dependentRelation(state, s0, true); + final String s0rcl = dependentRelation(state, s0, false); + final String s1rcl = dependentRelation(state, s1, false); + + final List features = new ArrayList<>(36); features.add("s0w=" + s0w); features.add("s0t=" + s0t); features.add("s1w=" + s1w); @@ -73,17 +83,36 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag features.add("s2t=" + s2t); features.add("b0w=" + b0w); features.add("b0t=" + b0t); + features.add("b1w=" + b1w); features.add("b1t=" + b1t); features.add("b2t=" + b2t); features.add("s0wt=" + s0w + '/' + s0t); + features.add("s1wt=" + s1w + '/' + s1t); + features.add("b0wt=" + b0w + '/' + b0t); features.add("s0w,b0w=" + s0w + '|' + b0w); features.add("s0t,b0t=" + s0t + '|' + b0t); + features.add("s0w,b0t=" + s0w + '|' + b0t); + features.add("s0t,b0w=" + s0t + '|' + b0w); + features.add("s0wt,b0t=" + s0w + '/' + s0t + '|' + b0t); features.add("s1t,s0t=" + s1t + '|' + s0t); + features.add("s1t,s0w=" + s1t + '|' + s0w); + features.add("s1w,s0t=" + s1w + '|' + s0t); features.add("s1t,s0t,b0t=" + s1t + '|' + s0t + '|' + b0t); features.add("s0t,b0t,b1t=" + s0t + '|' + b0t + '|' + b1t); + features.add("s2t,s1t,s0t=" + s2t + '|' + s1t + '|' + s0t); + features.add("s0lct=" + s0lct); + features.add("s0rct=" + s0rct); + features.add("s1lct=" + s1lct); + features.add("s1rct=" + s1rct); + features.add("s0lcl=" + s0lcl); + features.add("s0rcl=" + s0rcl); + features.add("s1rcl=" + s1rcl); + features.add("s1t,s1rct,s0t=" + s1t + '|' + s1rct + '|' + s0t); + features.add("s0t,s0lct,b0t=" + s0t + '|' + s0lct + '|' + b0t); features.add("s0deps=" + dependents(state, s0)); features.add("s1deps=" + dependents(state, s1)); features.add("dist=" + distance(s0, b0)); + features.add("dist,s0t,b0t=" + distance(s0, b0) + '|' + s0t + '|' + b0t); return features.toArray(new String[0]); } @@ -101,6 +130,32 @@ private static String tag(String[] tags, int index) { return index == ArcStandardState.NONE ? NONE_VALUE : tags[index]; } + /** The tag of a token's leftmost or rightmost dependent attached so far. */ + private static String dependentTag(ArcStandardState state, String[] tags, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + return tag(tags, dependent); + } + + /** The relation of a token's leftmost or rightmost dependent attached so far. */ + private static String dependentRelation(ArcStandardState state, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + if (dependent < 0) { + return NONE_VALUE; + } + final String relation = state.assignedRelation(dependent); + return relation == null ? NONE_VALUE : relation; + } + private static String dependents(ArcStandardState state, int index) { return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), 3)); } From 282e886a5fdaa8a1e529d7ba36888eb99f860c45 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 06:53:14 -0400 Subject: [PATCH 66/92] depparse: Pure-Java feedforward neural tier, training and inference in the JVM Adds the neural transition parser as plain array arithmetic with no native runtime: FeedforwardDependencyModel holds embeddings for words, tags, and arc labels, one cube-activation hidden layer, and a transition output layer in a versioned binary format of our own; FeedforwardDependencyParser decodes greedily with applicability masking; and FeedforwardDependencyTrainer trains the whole network in Java with minibatch AdaGrad over softmax cross-entropy, inverted dropout, a learned unknown word embedding, and a fixed seed for reproducibility. The feature template embeds fourteen configuration positions with second-order children, words and tags for all, labels for the dependent positions. First run on UD English EWT with gold UPOS tags and default settings: UAS 0.8585 and LAS 0.8351 over the full 25094-token test set, above the best classical result (0.8579 and 0.8316 from quasi-Newton training) with 13 minutes of training against 38, parsing at roughly 3.5k tokens per second single-threaded. Untapped levers: pretrained embedding initialization from the static embedding tables, capacity and schedule tuning, and beam search. (cherry picked from commit 7425d11a3ec40c057368be6e5b5dc71b11d6a5ba) --- .../tools/depparse/FeedforwardContext.java | 86 ++++ .../depparse/FeedforwardDependencyModel.java | 327 +++++++++++++ .../depparse/FeedforwardDependencyParser.java | 91 ++++ .../FeedforwardDependencyTrainer.java | 454 ++++++++++++++++++ .../FeedforwardDependencyParserTest.java | 141 ++++++ 5 files changed, 1099 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java new file mode 100644 index 0000000000..10dbcba958 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The feature template of the feedforward parser: a fixed set of configuration + * positions whose words, tags, and arc labels are embedded and concatenated into the + * network input. + * + *

Positions: the top three stack and buffer items; the leftmost and rightmost + * dependents of the top two stack items; and the leftmost dependent of the leftmost + * dependent and rightmost dependent of the rightmost dependent of the top two stack + * items, capturing second-order structure. Words and tags are read for all positions, + * labels only for the dependent positions, whose relations are already assigned.

+ */ +final class FeedforwardContext { + + /** The number of positions whose word and tag are embedded. */ + static final int POSITIONS = 14; + + /** The number of dependent positions whose arc label is embedded. */ + static final int LABEL_POSITIONS = 8; + + private FeedforwardContext() { + // static template only + } + + /** + * Extracts the symbolic features of a configuration: {@link #POSITIONS} words, then + * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent positions + * yield {@code null} entries, which the vocabulary maps to its padding symbol. + */ + static String[] extract(ArcStandardState state, String[] tokens, String[] tags) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int[] positions = { + s0, s1, state.stack(2), + state.buffer(0), state.buffer(1), state.buffer(2), + leftmost(state, s0), rightmost(state, s0), + leftmost(state, s1), rightmost(state, s1), + leftmost(state, leftmost(state, s0)), rightmost(state, rightmost(state, s0)), + leftmost(state, leftmost(state, s1)), rightmost(state, rightmost(state, s1)) + }; + final String[] features = new String[2 * POSITIONS + LABEL_POSITIONS]; + for (int i = 0; i < POSITIONS; i++) { + features[i] = symbol(tokens, positions[i]); + features[POSITIONS + i] = symbol(tags, positions[i]); + } + for (int i = 0; i < LABEL_POSITIONS; i++) { + final int position = positions[6 + i]; + features[2 * POSITIONS + i] = + position >= 0 ? state.assignedRelation(position) : null; + } + return features; + } + + private static int leftmost(ArcStandardState state, int index) { + return index >= 0 ? state.leftmostDependent(index) : ArcStandardState.NONE; + } + + private static int rightmost(ArcStandardState state, int index) { + return index >= 0 ? state.rightmostDependent(index) : ArcStandardState.NONE; + } + + private static String symbol(String[] values, int index) { + if (index == ArcStandardState.ROOT) { + return "*ROOT*"; + } + return index == ArcStandardState.NONE ? null : values[index]; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java new file mode 100644 index 0000000000..e1457efcab --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/** + * The weights of the feedforward transition parser: embeddings for words, tags, and arc + * labels, one hidden layer with cube activation, and a transition output layer, stored + * in a plain versioned binary format with no serialization framework involved. + * + *

This is the pure-Java neural tier: the network is executed with ordinary array + * arithmetic, so parsing needs no native runtime, and the same class scores + * configurations for training and decoding. Unknown words fall back to a learned + * unknown symbol; words are matched case-insensitively. Instances are immutable and + * safe to share between threads.

+ * + * @see FeedforwardDependencyParser + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +public class FeedforwardDependencyModel { + + private static final String MAGIC = "ONLP-FFDP-1"; + + static final String UNKNOWN = "*UNK*"; + static final String ABSENT = "*NULL*"; + + private final Map wordIds; + private final Map tagIds; + private final Map labelIds; + private final String[] transitions; + + private final int embeddingSize; + private final float[][] embeddings; + private final float[][] hiddenWeights; + private final float[] hiddenBias; + private final float[][] outputWeights; + private final float[] outputBias; + + FeedforwardDependencyModel(Map wordIds, Map tagIds, + Map labelIds, String[] transitions, int embeddingSize, + float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias, + float[][] outputWeights, float[] outputBias) { + this.wordIds = wordIds; + this.tagIds = tagIds; + this.labelIds = labelIds; + this.transitions = transitions; + this.embeddingSize = embeddingSize; + this.embeddings = embeddings; + this.hiddenWeights = hiddenWeights; + this.hiddenBias = hiddenBias; + this.outputWeights = outputWeights; + this.outputBias = outputBias; + } + + /** + * Scores every transition for a configuration described by embedding row indices. + * + * @param features The embedding rows of the configuration, as produced by + * {@link #featureIds(String[])}. Must not be {@code null}. + * @return One unnormalized score per transition, indexed like + * {@link #transitions()}. Never {@code null}. + */ + public double[] score(int[] features) { + final int hidden = hiddenBias.length; + final double[] h = new double[hidden]; + for (int j = 0; j < hidden; j++) { + final float[] row = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int f = 0; f < features.length; f++) { + final float[] embedding = embeddings[features[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + sum += row[offset + d] * embedding[d]; + } + } + h[j] = sum * sum * sum; + } + final double[] scores = new double[transitions.length]; + for (int o = 0; o < scores.length; o++) { + final float[] row = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hidden; j++) { + sum += row[j] * h[j]; + } + scores[o] = sum; + } + return scores; + } + + /** + * Maps the symbolic features of {@link FeedforwardContext} onto embedding rows. + * + * @param symbols The symbolic features. Must not be {@code null}. + * @return The embedding row per feature. Never {@code null}. + */ + public int[] featureIds(String[] symbols) { + final int[] ids = new int[symbols.length]; + for (int i = 0; i < FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(wordIds, normalize(symbols[i])); + } + for (int i = FeedforwardContext.POSITIONS; i < 2 * FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(tagIds, symbols[i]); + } + for (int i = 2 * FeedforwardContext.POSITIONS; i < symbols.length; i++) { + ids[i] = lookup(labelIds, symbols[i]); + } + return ids; + } + + /** + * @return The transition outcome strings by output index. Never {@code null}. + */ + public String[] transitions() { + return transitions.clone(); + } + + /** Lowercases a word symbol; special symbols and absences pass through. */ + static String normalize(String word) { + if (word == null) { + return null; + } + return word.startsWith("*") ? word : word.toLowerCase(Locale.ROOT); + } + + private static int lookup(Map ids, String symbol) { + Integer id = ids.get(symbol == null ? ABSENT : symbol); + if (id == null) { + id = ids.get(UNKNOWN); + } + return id; + } + + /** + * Writes the model in the versioned binary format. + * + * @param out The stream to write to. Must not be {@code null}. Not closed. + * @throws IOException Thrown if writing fails. + */ + public void serialize(OutputStream out) throws IOException { + if (out == null) { + throw new IllegalArgumentException("out must not be null"); + } + final DataOutputStream data = new DataOutputStream(new BufferedOutputStream(out)); + data.writeUTF(MAGIC); + writeVocabulary(data, wordIds); + writeVocabulary(data, tagIds); + writeVocabulary(data, labelIds); + data.writeInt(transitions.length); + for (final String transition : transitions) { + data.writeUTF(transition); + } + data.writeInt(embeddingSize); + writeMatrix(data, embeddings); + writeMatrix(data, hiddenWeights); + writeVector(data, hiddenBias); + writeMatrix(data, outputWeights); + writeVector(data, outputBias); + data.flush(); + } + + /** + * Loads a model from the versioned binary format. + * + * @param in The stream to read from. Must not be {@code null}. Not closed. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is not this format. + */ + public static FeedforwardDependencyModel load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final DataInputStream data = new DataInputStream(new BufferedInputStream(in)); + final String magic = data.readUTF(); + if (!MAGIC.equals(magic)) { + throw new IOException("not a feedforward dependency model: " + magic); + } + final Map wordIds = readVocabulary(data); + final Map tagIds = readVocabulary(data); + final Map labelIds = readVocabulary(data); + final String[] transitions = new String[data.readInt()]; + for (int i = 0; i < transitions.length; i++) { + transitions[i] = data.readUTF(); + } + final int embeddingSize = data.readInt(); + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions, + embeddingSize, readMatrix(data), readMatrix(data), readVector(data), + readMatrix(data), readVector(data)); + } + + /** + * Loads a model from a file. + * + * @param path The file to read. Must not be {@code null}. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is not this format. + */ + public static FeedforwardDependencyModel load(Path path) throws IOException { + if (path == null) { + throw new IllegalArgumentException("path must not be null"); + } + try (InputStream in = Files.newInputStream(path)) { + return load(in); + } + } + + private static void writeVocabulary(DataOutputStream data, Map ids) + throws IOException { + data.writeInt(ids.size()); + for (final Map.Entry entry : ids.entrySet()) { + data.writeUTF(entry.getKey()); + data.writeInt(entry.getValue()); + } + } + + private static Map readVocabulary(DataInputStream data) + throws IOException { + final int size = data.readInt(); + final Map ids = new HashMap<>(size * 2); + for (int i = 0; i < size; i++) { + final String symbol = data.readUTF(); + ids.put(symbol, data.readInt()); + } + return ids; + } + + private static void writeMatrix(DataOutputStream data, float[][] matrix) + throws IOException { + data.writeInt(matrix.length); + data.writeInt(matrix.length == 0 ? 0 : matrix[0].length); + for (final float[] row : matrix) { + for (final float value : row) { + data.writeFloat(value); + } + } + } + + private static float[][] readMatrix(DataInputStream data) throws IOException { + final int rows = data.readInt(); + final int columns = data.readInt(); + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = data.readFloat(); + } + } + return matrix; + } + + private static void writeVector(DataOutputStream data, float[] vector) throws IOException { + data.writeInt(vector.length); + for (final float value : vector) { + data.writeFloat(value); + } + } + + private static float[] readVector(DataInputStream data) throws IOException { + final float[] vector = new float[data.readInt()]; + for (int i = 0; i < vector.length; i++) { + vector[i] = data.readFloat(); + } + return vector; + } + + Map wordIds() { + return wordIds; + } + + Map tagIds() { + return tagIds; + } + + Map labelIds() { + return labelIds; + } + + int embeddingSize() { + return embeddingSize; + } + + float[][] embeddings() { + return embeddings; + } + + float[][] hiddenWeights() { + return hiddenWeights; + } + + float[] hiddenBias() { + return hiddenBias; + } + + float[][] outputWeights() { + return outputWeights; + } + + float[] outputBias() { + return outputBias; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java new file mode 100644 index 0000000000..aaae83cdfa --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The pure-Java neural {@link DependencyParser}: a greedy arc-standard decoder over the + * {@link FeedforwardDependencyModel}, picking the highest scoring applicable transition + * for each configuration. + * + *

Inference is ordinary array arithmetic with no native runtime involved, so this + * parser deploys exactly like the classical one while scoring configurations with + * learned dense representations instead of sparse feature conjunctions.

+ * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +public class FeedforwardDependencyParser implements DependencyParser { + + private final FeedforwardDependencyModel model; + private final Transition[] transitions; + + /** + * Initializes a {@link FeedforwardDependencyParser}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model; + final String[] outcomes = model.transitions(); + this.transitions = new Transition[outcomes.length]; + for (int i = 0; i < outcomes.length; i++) { + transitions[i] = Transition.decode(outcomes[i]); + } + } + + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + final double[] scores = model.score( + model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + Transition best = null; + double bestScore = Double.NEGATIVE_INFINITY; + for (int i = 0; i < scores.length; i++) { + if (scores[i] > bestScore && state.canApply(transitions[i])) { + best = transitions[i]; + bestScore = scores[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + state.apply(best); + } + return state.toGraph(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java new file mode 100644 index 0000000000..622e05c05d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.util.ObjectStream; + +/** + * Trains the {@link FeedforwardDependencyModel} entirely in Java: oracle-derived + * transition examples, minibatch AdaGrad over a softmax cross-entropy loss, cube + * activation, and inverted dropout on the hidden layer. No external training framework + * is involved, so the whole neural tier, training and inference, is plain array + * arithmetic inside the JVM. + * + *

Words below the frequency cutoff share a learned unknown embedding; absent + * template positions share a learned padding embedding. Non-projective samples have no + * arc-standard derivation and are skipped. Training is deterministic for a fixed + * {@link Settings#seed()}.

+ * + * @since 3.0.0 + */ +public final class FeedforwardDependencyTrainer { + + private static final Logger logger = + LoggerFactory.getLogger(FeedforwardDependencyTrainer.class); + + private static final double ADAGRAD_EPSILON = 1e-6; + + private FeedforwardDependencyTrainer() { + // static trainer only + } + + /** + * The training hyperparameters. + * + * @param embeddingSize The embedding dimensionality. Must be positive. + * @param hiddenSize The hidden layer width. Must be positive. + * @param epochs The number of passes over the examples. Must be positive. + * @param batchSize The minibatch size. Must be positive. + * @param learningRate The AdaGrad step size. Must be positive. + * @param l2 The L2 penalty applied to the dense weights. Must not be negative. + * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}. + * @param wordCutoff The minimum frequency for a word to get its own embedding. Must + * not be negative. + * @param seed The random seed making a run reproducible. + */ + public record Settings(int embeddingSize, int hiddenSize, int epochs, int batchSize, + double learningRate, double l2, double dropout, int wordCutoff, long seed) { + + /** + * Validates the hyperparameters. + * + * @throws IllegalArgumentException Thrown if a value is outside its documented + * range. + */ + public Settings { + if (embeddingSize <= 0 || hiddenSize <= 0 || epochs <= 0 || batchSize <= 0) { + throw new IllegalArgumentException("sizes, epochs and batch must be positive"); + } + if (learningRate <= 0.0 || l2 < 0.0) { + throw new IllegalArgumentException("learningRate must be positive, l2 not negative"); + } + if (!(dropout >= 0.0 && dropout < 1.0)) { + throw new IllegalArgumentException("dropout must be in [0, 1): " + dropout); + } + if (wordCutoff < 0) { + throw new IllegalArgumentException("wordCutoff must not be negative"); + } + } + + /** + * @return The default hyperparameters. Never {@code null}. + */ + public static Settings defaults() { + return new Settings(50, 200, 10, 256, 0.02, 1e-8, 0.5, 2, 17L); + } + } + + /** + * Trains a model from dependency samples. + * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or no + * trainable example can be derived from the samples. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings) throws IOException { + if (samples == null || settings == null) { + throw new IllegalArgumentException("samples and settings must not be null"); + } + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + final FeedforwardDependencyModel model = initialize(corpus, settings); + final List featureList = new ArrayList<>(); + final List goldList = new ArrayList<>(); + collectExamples(corpus, model, featureList, goldList); + if (featureList.isEmpty()) { + throw new IllegalArgumentException("no trainable examples in the samples"); + } + optimize(model, featureList, goldList, settings); + return model; + } + + /** Builds the vocabularies and randomly initialized weights. */ + private static FeedforwardDependencyModel initialize(List corpus, + Settings settings) { + final Map wordCounts = new HashMap<>(); + final Map tagIds = new HashMap<>(); + final Map labelIds = new HashMap<>(); + final Map transitionIds = new HashMap<>(); + for (final DependencySample s : corpus) { + for (final String token : s.getTokens()) { + wordCounts.merge(FeedforwardDependencyModel.normalize(token), 1, Integer::sum); + } + for (final String tag : s.getTags()) { + tagIds.putIfAbsent(tag, 0); + } + final DependencyGraph graph = s.getGraph(); + for (int i = 0; i < graph.size(); i++) { + labelIds.putIfAbsent(graph.relationOf(i), 0); + } + } + // the outcome space: shift plus both arc directions for every observed label + transitionIds.putIfAbsent(Transition.SHIFT.encode(), 0); + for (final String label : labelIds.keySet()) { + transitionIds.putIfAbsent(Transition.leftArc(label).encode(), 0); + transitionIds.putIfAbsent(Transition.rightArc(label).encode(), 0); + } + + int row = 0; + final Map wordIds = new HashMap<>(); + for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, "*ROOT*")) { + wordIds.put(special, row++); + } + for (final Map.Entry entry : wordCounts.entrySet()) { + if (entry.getValue() >= settings.wordCutoff() && !wordIds.containsKey(entry.getKey())) { + wordIds.put(entry.getKey(), row++); + } + } + final Map tags = new HashMap<>(); + for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, "*ROOT*")) { + tags.put(special, row++); + } + for (final String tag : tagIds.keySet()) { + tags.put(tag, row++); + } + final Map labels = new HashMap<>(); + for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT)) { + labels.put(special, row++); + } + for (final String label : labelIds.keySet()) { + labels.put(label, row++); + } + + int transitionIndex = 0; + final String[] transitions = new String[transitionIds.size()]; + for (final String encoded : transitionIds.keySet()) { + transitions[transitionIndex] = encoded; + transitionIds.put(encoded, transitionIndex++); + } + + final Random random = new Random(settings.seed()); + final int inputSize = + (2 * FeedforwardContext.POSITIONS + FeedforwardContext.LABEL_POSITIONS) + * settings.embeddingSize(); + final float[][] embeddings = uniform(random, row, settings.embeddingSize(), 0.01); + final float[][] hiddenWeights = uniform(random, settings.hiddenSize(), inputSize, + Math.sqrt(6.0 / (inputSize + settings.hiddenSize()))); + final float[][] outputWeights = uniform(random, transitions.length, + settings.hiddenSize(), + Math.sqrt(6.0 / (settings.hiddenSize() + transitions.length))); + return new FeedforwardDependencyModel(wordIds, tags, labels, transitions, + settings.embeddingSize(), embeddings, hiddenWeights, + new float[settings.hiddenSize()], outputWeights, new float[transitions.length]); + } + + /** Replays the oracle over every projective sample, emitting one example per step. */ + private static void collectExamples(List corpus, + FeedforwardDependencyModel model, List featureList, List goldList) { + final Map transitionIds = new HashMap<>(); + final String[] transitions = model.transitions(); + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(transitions[i], i); + } + int skipped = 0; + for (final DependencySample sample : corpus) { + final List oracle; + try { + oracle = ArcStandardOracle.transitions(sample.getGraph()); + } catch (IllegalArgumentException e) { + skipped++; + continue; + } + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : oracle) { + featureList.add(model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + goldList.add(transitionIds.get(transition.encode())); + state.apply(transition); + } + } + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + } + } + + /** Minibatch AdaGrad over softmax cross-entropy with cube activation and dropout. */ + private static void optimize(FeedforwardDependencyModel model, List featureList, + List goldList, Settings settings) { + final int exampleCount = featureList.size(); + final int[][] features = featureList.toArray(new int[0][]); + final int[] gold = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + gold[i] = goldList.get(i); + } + + final float[][] embeddings = model.embeddings(); + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + final int embeddingSize = settings.embeddingSize(); + final int hiddenSize = settings.hiddenSize(); + final int outputSize = outputBias.length; + final int inputSize = features[0].length * embeddingSize; + + final double[][] embeddingAccumulator = + new double[embeddings.length][embeddingSize]; + final double[][] hiddenAccumulator = new double[hiddenSize][inputSize]; + final double[] hiddenBiasAccumulator = new double[hiddenSize]; + final double[][] outputAccumulator = new double[outputSize][hiddenSize]; + final double[] outputBiasAccumulator = new double[outputSize]; + + final double[][] hiddenGradient = new double[hiddenSize][inputSize]; + final double[] hiddenBiasGradient = new double[hiddenSize]; + final double[][] outputGradient = new double[outputSize][hiddenSize]; + final double[] outputBiasGradient = new double[outputSize]; + final Map embeddingGradients = new HashMap<>(); + + final Random random = new Random(settings.seed()); + final int[] order = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + order[i] = i; + } + + final double keep = 1.0 - settings.dropout(); + final double[] x = new double[inputSize]; + final double[] pre = new double[hiddenSize]; + final double[] hidden = new double[hiddenSize]; + final boolean[] mask = new boolean[hiddenSize]; + final double[] probabilities = new double[outputSize]; + final double[] hiddenDelta = new double[hiddenSize]; + final double[] inputDelta = new double[inputSize]; + + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + for (int batchStart = 0; batchStart < exampleCount; + batchStart += settings.batchSize()) { + final int batchEnd = Math.min(batchStart + settings.batchSize(), exampleCount); + final int batch = batchEnd - batchStart; + zero(hiddenGradient); + java.util.Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + java.util.Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + + for (int b = batchStart; b < batchEnd; b++) { + final int[] feats = features[order[b]]; + final int goldTransition = gold[order[b]]; + for (int f = 0; f < feats.length; f++) { + final float[] embedding = embeddings[feats[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + for (int j = 0; j < hiddenSize; j++) { + mask[j] = random.nextDouble() < keep; + if (!mask[j]) { + pre[j] = 0.0; + hidden[j] = 0.0; + continue; + } + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum / keep; + } + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + max = Math.max(max, sum); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + for (int o = 0; o < outputSize; o++) { + probabilities[o] /= normalizer; + } + loss -= Math.log(Math.max(probabilities[goldTransition], 1e-12)); + + java.util.Arrays.fill(hiddenDelta, 0.0); + java.util.Arrays.fill(inputDelta, 0.0); + for (int o = 0; o < outputSize; o++) { + final double delta = probabilities[o] - (o == goldTransition ? 1.0 : 0.0); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + for (int j = 0; j < hiddenSize; j++) { + if (!mask[j]) { + continue; + } + final double preDelta = hiddenDelta[j] * 3.0 * pre[j] * pre[j] / keep; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < feats.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(feats[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + update(hiddenWeights, hiddenGradient, hiddenAccumulator, batch, settings); + updateVector(hiddenBias, hiddenBiasGradient, hiddenBiasAccumulator, batch, settings); + update(outputWeights, outputGradient, outputAccumulator, batch, settings); + updateVector(outputBias, outputBiasGradient, outputBiasAccumulator, batch, settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = embeddings[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d] / batch; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + } + logger.info("epoch {}: loss {} in {} ms", epoch, loss / exampleCount, + System.currentTimeMillis() - epochStart); + } + } + + private static void update(float[][] weights, double[][] gradients, + double[][] accumulators, int batch, Settings settings) { + for (int r = 0; r < weights.length; r++) { + final float[] weightRow = weights[r]; + final double[] gradientRow = gradients[r]; + final double[] accumulatorRow = accumulators[r]; + for (int c = 0; c < weightRow.length; c++) { + final double gradient = gradientRow[c] / batch + settings.l2() * weightRow[c]; + accumulatorRow[c] += gradient * gradient; + weightRow[c] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[c]) + ADAGRAD_EPSILON); + } + } + } + + private static void updateVector(float[] weights, double[] gradients, + double[] accumulators, int batch, Settings settings) { + for (int i = 0; i < weights.length; i++) { + final double gradient = gradients[i] / batch; + accumulators[i] += gradient * gradient; + weights[i] -= settings.learningRate() * gradient + / (Math.sqrt(accumulators[i]) + ADAGRAD_EPSILON); + } + } + + private static float[][] uniform(Random random, int rows, int columns, double scale) { + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = (float) ((random.nextDouble() * 2.0 - 1.0) * scale); + } + } + return matrix; + } + + private static void zero(double[][] matrix) { + for (final double[] row : matrix) { + java.util.Arrays.fill(row, 0.0); + } + } + + private static void shuffle(int[] order, Random random) { + for (int i = order.length - 1; i > 0; i--) { + final int j = random.nextInt(i + 1); + final int swap = order[i]; + order[i] = order[j]; + order[j] = swap; + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java new file mode 100644 index 0000000000..253c55c48d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStreamUtils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the pure-Java neural tier end to end: training on a tiny corpus must let the + * greedy feedforward parser reproduce the training sentences, and a model must survive + * the serialization round trip bit-for-bit in behavior. + */ +public class FeedforwardDependencyParserTest { + + private static FeedforwardDependencyModel model; + private static FeedforwardDependencyParser parser; + + private static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + private static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + @BeforeAll + static void trainParser() throws IOException { + // dropout off so the tiny network memorizes deterministically + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 120, 32, 0.05, 0.0, 0.0, 1, 17L); + model = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + parser = new FeedforwardDependencyParser(model); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + @Test + void testUnknownWordsStillYieldASingleRootedTree() { + final DependencyGraph parsed = parser.parse(new String[] {"unseen", "words"}, + new String[] {"JJ", "NNS"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + final FeedforwardDependencyModel reloaded = + FeedforwardDependencyModel.load(new ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new FeedforwardDependencyParser(reloaded) + .parse(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), parsed); + } + + @Test + void testCorruptModelFailsLoud() { + assertThrows(IOException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream("not a model".getBytes()))); + } + + @Test + void testSettingsValidation() { + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(0, 32, 10, 32, 0.05, 0.0, 0.0, 1, 17L)); + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(16, 32, 10, 32, -1.0, 0.0, 0.0, 1, 17L)); + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(16, 32, 10, 32, 0.05, 0.0, 1.0, 1, 17L)); + } + + @Test + void testArgumentValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null)); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train(null, + FeedforwardDependencyTrainer.Settings.defaults())); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } +} From 70987c15de5ae2066b2524b7b7ae1374c21d2576 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 06:58:34 -0400 Subject: [PATCH 67/92] depparse: Pretrained word embedding seeding for the feedforward trainer Adds a training overload taking a pretrained vector provider: vocabulary words the provider knows start from their pretrained vectors instead of random noise, stay trainable, and everything the provider does not know keeps the random initialization. The provider is a training-time ingredient only, since the learned embeddings ship inside the model, so parsing carries no dependency on the embedding source. This is the seam that lets the static embedding tables feed the parser. (cherry picked from commit 0084cf36f6fb74a634fe6d0e422c743f2180aa15) --- .../FeedforwardDependencyTrainer.java | 54 +++++++++++++++++++ .../FeedforwardDependencyParserTest.java | 19 +++++++ 2 files changed, 73 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index 622e05c05d..f0ae0d4b91 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -112,6 +112,33 @@ public static Settings defaults() { */ public static FeedforwardDependencyModel train(ObjectStream samples, Settings settings) throws IOException { + return train(samples, settings, null); + } + + /** + * Trains a model from dependency samples, seeding word embeddings from a pretrained + * source. + * + *

The provider is consulted once per vocabulary word during initialization; words + * it returns {@code null} for keep their random initialization, and all embeddings + * remain trainable afterwards. The pretrained source is a training-time ingredient + * only: the learned embeddings ship inside the model, so parsing carries no + * dependency on the source.

+ * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @param pretrained Maps a normalized word to its pretrained vector of exactly + * {@link Settings#embeddingSize()} dimensions, or {@code null} for + * unknown words. May be {@code null} to disable seeding. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if {@code samples} or {@code settings} is + * {@code null}, no trainable example can be derived, or a pretrained vector + * has the wrong dimensionality. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings, java.util.function.Function pretrained) + throws IOException { if (samples == null || settings == null) { throw new IllegalArgumentException("samples and settings must not be null"); } @@ -121,6 +148,9 @@ public static FeedforwardDependencyModel train(ObjectStream sa corpus.add(sample); } final FeedforwardDependencyModel model = initialize(corpus, settings); + if (pretrained != null) { + seed(model, pretrained, settings); + } final List featureList = new ArrayList<>(); final List goldList = new ArrayList<>(); collectExamples(corpus, model, featureList, goldList); @@ -131,6 +161,30 @@ public static FeedforwardDependencyModel train(ObjectStream sa return model; } + /** Overwrites the random word rows with pretrained vectors where available. */ + private static void seed(FeedforwardDependencyModel model, + java.util.function.Function pretrained, Settings settings) { + int seeded = 0; + for (final Map.Entry entry : model.wordIds().entrySet()) { + if (entry.getKey().startsWith("*")) { + continue; // the special symbols have no pretrained meaning + } + final float[] vector = pretrained.apply(entry.getKey()); + if (vector == null) { + continue; + } + if (vector.length != settings.embeddingSize()) { + throw new IllegalArgumentException("pretrained vector for '" + entry.getKey() + + "' has " + vector.length + " dimensions, expected " + + settings.embeddingSize()); + } + System.arraycopy(vector, 0, model.embeddings()[entry.getValue()], 0, vector.length); + seeded++; + } + logger.info("seeded {} of {} word embeddings from the pretrained source", seeded, + model.wordIds().size()); + } + /** Builds the vocabularies and randomly initialized weights. */ private static FeedforwardDependencyModel initialize(List corpus, Settings settings) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 253c55c48d..6f762faeb4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -123,6 +123,25 @@ void testSettingsValidation() { .Settings(16, 32, 10, 32, 0.05, 0.0, 1.0, 1, 17L)); } + @Test + void testPretrainedSeedingAppliesAndValidates() throws IOException { + // near-zero learning keeps the seeded row observable after one epoch + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(4, 8, 1, 32, 1e-9, 0.0, 0.0, 1, 17L); + final float[] vector = {0.25f, -0.5f, 0.75f, -1.0f}; + final FeedforwardDependencyModel seeded = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> "dog".equals(word) ? vector.clone() : null); + final int row = seeded.wordIds().get("dog"); + for (int d = 0; d < vector.length; d++) { + assertEquals(vector[d], seeded.embeddings()[row][d], 1e-4); + } + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> new float[] {1.0f})); + } + @Test void testArgumentValidation() { assertThrows(IllegalArgumentException.class, From b0a3ed6b38c7a574a3908409dd4419f1b37cbe52 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 09:34:26 -0400 Subject: [PATCH 68/92] depparse: Beam search decoding for the feedforward parser The decoder keeps the best transition sequences side by side, scored by summed log-probabilities, so one locally attractive but globally wrong transition no longer commits the whole parse. Beam size one keeps the exact greedy fast path; every arc-standard derivation has the same length, so summed scores compare without normalization. (cherry picked from commit 2c3009983cf075f8f77c3ae88224368a68195dab) --- .../tools/depparse/ArcStandardState.java | 22 +++ .../depparse/FeedforwardDependencyParser.java | 133 +++++++++++++++++- .../FeedforwardDependencyParserTest.java | 37 +++++ 3 files changed, 188 insertions(+), 4 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index 0464a3436d..8f79eeff1f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -76,6 +76,28 @@ public ArcStandardState(int tokenCount) { java.util.Arrays.fill(this.rightmostDependents, NONE); } + private ArcStandardState(ArcStandardState source) { + this.tokenCount = source.tokenCount; + this.stack = source.stack.clone(); + this.heads = source.heads.clone(); + this.relations = source.relations.clone(); + this.assignedDependents = source.assignedDependents.clone(); + this.leftmostDependents = source.leftmostDependents.clone(); + this.rightmostDependents = source.rightmostDependents.clone(); + this.top = source.top; + this.bufferFront = source.bufferFront; + } + + /** + * Creates an independent copy of this configuration, so alternatives can be advanced + * side by side during search. + * + * @return A copy that can be advanced without affecting this state. Never {@code null}. + */ + public ArcStandardState copy() { + return new ArcStandardState(this); + } + /** * @return {@code true} if the buffer is empty and only the artificial root remains on * the stack, so the parse is complete. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index aaae83cdfa..0ada7c985a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -17,10 +17,20 @@ package opennlp.tools.depparse; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + /** - * The pure-Java neural {@link DependencyParser}: a greedy arc-standard decoder over the - * {@link FeedforwardDependencyModel}, picking the highest scoring applicable transition - * for each configuration. + * The pure-Java neural {@link DependencyParser}: an arc-standard decoder over the + * {@link FeedforwardDependencyModel}, greedy by default and beamed when constructed + * with a beam size above one. + * + *

With a beam, the decoder keeps the highest scoring transition sequences side by + * side, scored by summed log-probabilities, so one locally attractive but globally + * wrong transition no longer commits the whole parse. Every complete arc-standard + * derivation of a sentence has the same length, which keeps the summed scores + * comparable without length normalization.

* *

Inference is ordinary array arithmetic with no native runtime involved, so this * parser deploys exactly like the classical one while scoring configurations with @@ -36,19 +46,38 @@ public class FeedforwardDependencyParser implements DependencyParser { private final FeedforwardDependencyModel model; private final Transition[] transitions; + private final int beamSize; /** - * Initializes a {@link FeedforwardDependencyParser}. + * Initializes a greedy {@link FeedforwardDependencyParser}. * * @param model The model to parse with. Must not be {@code null}. * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an * outcome of the model does not decode to a transition. */ public FeedforwardDependencyParser(FeedforwardDependencyModel model) { + this(model, 1); + } + + /** + * Initializes a {@link FeedforwardDependencyParser} with a beam. + * + * @param model The model to parse with. Must not be {@code null}. + * @param beamSize The number of transition sequences to keep side by side. Must be + * greater than zero; {@code 1} decodes greedily. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}, + * {@code beamSize} is not positive, or an outcome of the model does not + * decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model, int beamSize) { if (model == null) { throw new IllegalArgumentException("model must not be null"); } + if (beamSize < 1) { + throw new IllegalArgumentException("beamSize must be positive: " + beamSize); + } this.model = model; + this.beamSize = beamSize; final String[] outcomes = model.transitions(); this.transitions = new Transition[outcomes.length]; for (int i = 0; i < outcomes.length; i++) { @@ -68,6 +97,20 @@ public DependencyGraph parse(String[] tokens, String[] tags) { throw new IllegalArgumentException("tokens and tags must have the same length: " + tokens.length + " != " + tags.length); } + if (beamSize == 1) { + return greedyParse(tokens, tags); + } + return beamParse(tokens, tags); + } + + /** + * Decodes greedily: the highest scoring applicable transition wins each step. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + */ + private DependencyGraph greedyParse(String[] tokens, String[] tags) { final ArcStandardState state = new ArcStandardState(tokens.length); while (!state.isTerminal()) { final double[] scores = model.score( @@ -88,4 +131,86 @@ public DependencyGraph parse(String[] tokens, String[] tags) { } return state.toGraph(); } + + /** One search alternative: a configuration, its summed log-probability score, and the + * transition that would advance it, {@code null} once complete. */ + private record Alternative(ArcStandardState state, double score, Transition next) { + } + + /** + * Decodes with a beam: the {@code beamSize} best transition sequences advance side by + * side and the best complete one wins. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + */ + private DependencyGraph beamParse(String[] tokens, String[] tags) { + List beam = + List.of(new Alternative(new ArcStandardState(tokens.length), 0.0, null)); + while (true) { + boolean advanced = false; + final List expansions = new ArrayList<>(); + for (final Alternative alternative : beam) { + if (alternative.state().isTerminal()) { + expansions.add(alternative); + continue; + } + advanced = true; + final double[] logProbabilities = logSoftmax(model.score( + model.featureIds(FeedforwardContext.extract(alternative.state(), tokens, tags)))); + for (int i = 0; i < logProbabilities.length; i++) { + if (alternative.state().canApply(transitions[i])) { + expansions.add(new Alternative(alternative.state(), + alternative.score() + logProbabilities[i], transitions[i])); + } + } + } + if (!advanced) { + break; + } + expansions.sort(Comparator.comparingDouble(Alternative::score).reversed()); + final List survivors = + new ArrayList<>(Math.min(beamSize, expansions.size())); + for (int i = 0; i < expansions.size() && survivors.size() < beamSize; i++) { + final Alternative expansion = expansions.get(i); + if (expansion.next() == null) { + survivors.add(expansion); + } else { + final ArcStandardState state = expansion.state().copy(); + state.apply(expansion.next()); + survivors.add(new Alternative(state, expansion.score(), null)); + } + } + if (survivors.isEmpty()) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in the beam"); + } + beam = survivors; + } + return beam.get(0).state().toGraph(); + } + + /** + * Normalizes raw transition scores to log-probabilities. + * + * @param scores The raw output scores. + * @return The log-softmax of {@code scores}. Never {@code null}. + */ + private static double[] logSoftmax(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = max + Math.log(sum); + final double[] logProbabilities = new double[scores.length]; + for (int i = 0; i < scores.length; i++) { + logProbabilities[i] = scores[i] - logSum; + } + return logProbabilities; + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 6f762faeb4..48bf25cfa3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -95,6 +95,43 @@ void testUnknownWordsStillYieldASingleRootedTree() { parsed.root(); } + @Test + void testBeamOfOneMatchesGreedy() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 1); + for (final DependencySample sample : corpus()) { + assertEquals(parser.parse(sample.getTokens(), sample.getTags()), + beamed.parse(sample.getTokens(), sample.getTags())); + } + } + + @Test + void testBeamedParserReproducesTrainingSentences() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 4); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + beamed.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + @Test + void testBeamedParseIsDeterministicAndSingleRooted() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 8); + final String[] tokens = {"unseen", "words", "everywhere"}; + final String[] tags = {"JJ", "NNS", "RB"}; + final DependencyGraph first = beamed.parse(tokens, tags); + assertEquals(first, beamed.parse(tokens, tags)); + assertEquals(3, first.size()); + first.root(); + } + + @Test + void testBeamSizeValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(model, 0)); + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null, 4)); + } + @Test void testModelRoundTripThroughSerialization() throws IOException { final ByteArrayOutputStream out = new ByteArrayOutputStream(); From 2be280921979ccb0f67b1cef67de213ed49ae621 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 10:33:46 -0400 Subject: [PATCH 69/92] depparse: Global refinement with beam search and early update After local training, sentences are re-decoded with a beam while the gold derivation is tracked through it; the moment gold falls out an early update pushes the model toward keeping it, under a conditional likelihood over the beam's candidate paths. Paths are scored exactly as the beamed parser scores them, so training optimizes the quantity decoding uses. Refinement mutates the model in place, deterministic for a fixed seed. (cherry picked from commit bd663ba9cb577ffb91e1e2ec89619bb5ad60c7ed) --- .../FeedforwardDependencyTrainer.java | 386 ++++++++++++++++++ .../FeedforwardDependencyParserTest.java | 28 ++ 2 files changed, 414 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index f0ae0d4b91..c676949ca9 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -161,6 +161,392 @@ public static FeedforwardDependencyModel train(ObjectStream sa return model; } + /** + * Fine-tunes a locally trained model globally: sentences are decoded with a beam, the + * gold derivation is tracked through it, and the moment the gold prefix falls out of + * the beam an early update pushes the model toward keeping it. The loss is a + * conditional likelihood over the beam's candidate paths, scored exactly like the + * beamed parser scores them, summed log-probabilities, so training optimizes the + * quantity decoding uses. + * + *

The model is updated in place with per-sentence AdaGrad steps and no dropout; + * {@link Settings#epochs()} counts the refinement passes. Refinement is deterministic + * for a fixed {@link Settings#seed()}. Parse afterwards with the same beam size.

+ * + * @param model The locally trained model to refine. Must not be {@code null}. + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters; {@code epochs}, {@code learningRate}, + * {@code l2}, and {@code seed} apply. Must not be {@code null}. + * @param beamSize The beam width to track the gold derivation in. Must be at least 2. + * @return The same model instance, refined. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code beamSize} is below 2, or no trainable sample can be derived. + */ + public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model, + ObjectStream samples, Settings settings, int beamSize) + throws IOException { + if (model == null || samples == null || settings == null) { + throw new IllegalArgumentException("model, samples and settings must not be null"); + } + if (beamSize < 2) { + throw new IllegalArgumentException("beamSize must be at least 2: " + beamSize); + } + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + final Map transitionIds = new HashMap<>(); + final Transition[] transitions = new Transition[model.transitions().length]; + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(model.transitions()[i], i); + transitions[i] = Transition.decode(model.transitions()[i]); + } + + final List trainable = new ArrayList<>(); + final List oracles = new ArrayList<>(); + for (final DependencySample s : corpus) { + final List oracle; + try { + oracle = ArcStandardOracle.transitions(s.getGraph()); + } catch (IllegalArgumentException e) { + continue; + } + final int[] encoded = new int[oracle.size()]; + for (int i = 0; i < encoded.length; i++) { + encoded[i] = transitionIds.get(oracle.get(i).encode()); + } + trainable.add(s); + oracles.add(encoded); + } + if (trainable.isEmpty()) { + throw new IllegalArgumentException("no trainable samples for refinement"); + } + + final GlobalOptimizer optimizer = new GlobalOptimizer(model, settings); + final Random random = new Random(settings.seed()); + final int[] order = new int[trainable.size()]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + int updates = 0; + for (final int index : order) { + final double sentenceLoss = optimizer.refineSentence(trainable.get(index), + oracles.get(index), transitions, beamSize); + if (sentenceLoss >= 0.0) { + loss += sentenceLoss; + updates++; + } + } + logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch, + loss / Math.max(updates, 1), updates, System.currentTimeMillis() - epochStart); + } + return model; + } + + /** One candidate path in the refinement beam: the parent link forms the history. */ + private static final class BeamNode { + private final BeamNode parent; + private final int[] features; + private final int transition; + private final double score; + private final boolean gold; + private ArcStandardState state; + + private BeamNode(BeamNode parent, int[] features, int transition, double score, + boolean gold) { + this.parent = parent; + this.features = features; + this.transition = transition; + this.score = score; + this.gold = gold; + } + } + + /** The forward, backward, and AdaGrad state for global refinement. */ + private static final class GlobalOptimizer { + private final FeedforwardDependencyModel model; + private final Settings settings; + private final int embeddingSize; + private final int hiddenSize; + private final int outputSize; + private final int inputSize; + + private final double[][] embeddingAccumulator; + private final double[][] hiddenAccumulator; + private final double[] hiddenBiasAccumulator; + private final double[][] outputAccumulator; + private final double[] outputBiasAccumulator; + + private final double[][] hiddenGradient; + private final double[] hiddenBiasGradient; + private final double[][] outputGradient; + private final double[] outputBiasGradient; + private final Map embeddingGradients = new HashMap<>(); + + private final double[] x; + private final double[] pre; + private final double[] hidden; + private final double[] probabilities; + private final double[] hiddenDelta; + private final double[] inputDelta; + + private GlobalOptimizer(FeedforwardDependencyModel model, Settings settings) { + this.model = model; + this.settings = settings; + this.embeddingSize = model.embeddings()[0].length; + this.hiddenSize = model.hiddenBias().length; + this.outputSize = model.outputBias().length; + this.inputSize = + (2 * FeedforwardContext.POSITIONS + FeedforwardContext.LABEL_POSITIONS) + * embeddingSize; + this.embeddingAccumulator = + new double[model.embeddings().length][embeddingSize]; + this.hiddenAccumulator = new double[hiddenSize][inputSize]; + this.hiddenBiasAccumulator = new double[hiddenSize]; + this.outputAccumulator = new double[outputSize][hiddenSize]; + this.outputBiasAccumulator = new double[outputSize]; + this.hiddenGradient = new double[hiddenSize][inputSize]; + this.hiddenBiasGradient = new double[hiddenSize]; + this.outputGradient = new double[outputSize][hiddenSize]; + this.outputBiasGradient = new double[outputSize]; + this.x = new double[inputSize]; + this.pre = new double[hiddenSize]; + this.hidden = new double[hiddenSize]; + this.probabilities = new double[outputSize]; + this.hiddenDelta = new double[hiddenSize]; + this.inputDelta = new double[inputSize]; + } + + /** + * Decodes one sentence with the beam, updating on the early-update point or the + * final beam. + * + * @param sample The sentence. + * @param oracle The gold transition indexes. + * @param transitions The decoded transition inventory. + * @param beamSize The beam width. + * @return The sentence loss, or {@code -1} when the sentence produced no update. + */ + private double refineSentence(DependencySample sample, int[] oracle, + Transition[] transitions, int beamSize) { + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + final BeamNode root = new BeamNode(null, null, -1, 0.0, true); + root.state = new ArcStandardState(tokens.length); + List beam = List.of(root); + + for (int step = 0; step < oracle.length; step++) { + final List expansions = new ArrayList<>(); + BeamNode goldChild = null; + for (final BeamNode node : beam) { + final int[] features = + model.featureIds(FeedforwardContext.extract(node.state, tokens, tags)); + forward(features); + logSoftmaxInPlace(probabilities); + for (int i = 0; i < outputSize; i++) { + if (node.state.canApply(transitions[i])) { + final boolean goldNext = node.gold && i == oracle[step]; + final BeamNode child = new BeamNode(node, features, i, + node.score + probabilities[i], goldNext); + expansions.add(child); + if (goldNext) { + goldChild = child; + } + } + } + } + expansions.sort((a, b) -> Double.compare(b.score, a.score)); + final List survivors = + new ArrayList<>(expansions.subList(0, Math.min(beamSize, expansions.size()))); + boolean goldSurvives = false; + for (final BeamNode survivor : survivors) { + if (survivor.gold) { + goldSurvives = true; + break; + } + } + if (!goldSurvives) { + if (goldChild == null) { + return -1.0; // the oracle transition was inapplicable; nothing to learn from + } + survivors.add(goldChild); + return updateFromCandidates(survivors); + } + if (step == oracle.length - 1) { + return updateFromCandidates(survivors); + } + for (final BeamNode survivor : survivors) { + survivor.state = survivor.parent.state.copy(); + survivor.state.apply(transitions[survivor.transition]); + } + beam = survivors; + } + return -1.0; + } + + /** Applies the conditional-likelihood update over the candidate paths. */ + private double updateFromCandidates(List candidates) { + double max = Double.NEGATIVE_INFINITY; + double goldScore = Double.NEGATIVE_INFINITY; + for (final BeamNode candidate : candidates) { + max = Math.max(max, candidate.score); + if (candidate.gold) { + goldScore = candidate.score; + } + } + double normalizer = 0.0; + for (final BeamNode candidate : candidates) { + normalizer += Math.exp(candidate.score - max); + } + final double logNormalizer = max + Math.log(normalizer); + + zero(hiddenGradient); + java.util.Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + java.util.Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + for (final BeamNode candidate : candidates) { + final double weight = Math.exp(candidate.score - logNormalizer) + - (candidate.gold ? 1.0 : 0.0); + if (weight == 0.0) { + continue; + } + for (BeamNode node = candidate; node.parent != null; node = node.parent) { + backward(node.features, node.transition, weight); + } + } + update(model.hiddenWeights(), hiddenGradient, hiddenAccumulator, 1, settings); + updateVector(model.hiddenBias(), hiddenBiasGradient, hiddenBiasAccumulator, 1, + settings); + update(model.outputWeights(), outputGradient, outputAccumulator, 1, settings); + updateVector(model.outputBias(), outputBiasGradient, outputBiasAccumulator, 1, + settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = model.embeddings()[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d]; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + return logNormalizer - goldScore; + } + + /** Computes hidden activations and raw output scores for one feature vector. */ + private void forward(int[] features) { + final float[][] embeddings = model.embeddings(); + for (int f = 0; f < features.length; f++) { + final float[] embedding = embeddings[features[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + for (int j = 0; j < hiddenSize; j++) { + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum; + } + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + } + } + + /** + * Accumulates gradients for one decoded step: the weighted difference between the + * step's softmax and its chosen transition. + * + * @param features The step's input features. + * @param chosen The transition the path took at this step. + * @param weight The path's weight in the candidate distribution. + */ + private void backward(int[] features, int chosen, double weight) { + forward(features); + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + max = Math.max(max, probabilities[o]); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + java.util.Arrays.fill(hiddenDelta, 0.0); + java.util.Arrays.fill(inputDelta, 0.0); + final float[][] outputWeights = model.outputWeights(); + for (int o = 0; o < outputSize; o++) { + // dL/dlogit for a path's step under the conditional likelihood: the path weight + // times how the step's log-probability responds to this logit + final double delta = + weight * ((o == chosen ? 1.0 : 0.0) - probabilities[o] / normalizer); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + for (int j = 0; j < hiddenSize; j++) { + final double preDelta = hiddenDelta[j] * 3.0 * pre[j] * pre[j]; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < features.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(features[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + /** Turns raw scores into log-probabilities in place. */ + private static void logSoftmaxInPlace(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = max + Math.log(sum); + for (int i = 0; i < scores.length; i++) { + scores[i] -= logSum; + } + } + } + /** Overwrites the random word rows with pretrained vectors where available. */ private static void seed(FeedforwardDependencyModel model, java.util.function.Function pretrained, Settings settings) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 48bf25cfa3..a4a8c347d8 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -124,6 +124,34 @@ void testBeamedParseIsDeterministicAndSingleRooted() { first.root(); } + @Test + void testRefinementKeepsToyPerformance() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final FeedforwardDependencyTrainer.Settings refineSettings = + new FeedforwardDependencyTrainer.Settings(16, 32, 2, 32, 0.01, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new FeedforwardDependencyParser(refined, 2)); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + @Test + void testRefineValidation() { + final FeedforwardDependencyTrainer.Settings settings = + FeedforwardDependencyTrainer.Settings.defaults(); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(null, ObjectStreamUtils.createObjectStream(corpus()), settings, 4)); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(model, ObjectStreamUtils.createObjectStream(corpus()), settings, 1)); + } + @Test void testBeamSizeValidation() { assertThrows(IllegalArgumentException.class, From d1334ad278332915daa6650545dc294e18c7f814 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:21:13 -0400 Subject: [PATCH 70/92] depparse: State, transition, usage, and edge-case tests; expanded trainer and parser javadoc --- .../ConlluDependencyParserUsageTest.java | 178 ++++++++++++ .../ConlluDependencySampleStreamTest.java | 35 +++ .../tools/depparse/ArcStandardOracle.java | 2 +- .../tools/depparse/FeedforwardContext.java | 2 +- .../depparse/FeedforwardDependencyParser.java | 8 +- .../FeedforwardDependencyTrainer.java | 13 +- .../tools/depparse/ArcStandardStateTest.java | 148 ++++++++++ .../DependencyParserEdgeCaseTest.java | 264 ++++++++++++++++++ .../tools/depparse/TransitionTest.java | 73 +++++ 9 files changed, 713 insertions(+), 10 deletions(-) create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java new file mode 100644 index 0000000000..960653a778 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Demonstrates the full dependency parsing workflow on a self-contained fixture: read + * gold sentences from CoNLL-U content, train a {@link DependencyParserME}, parse a + * sentence, inspect the resulting {@link DependencyGraph}, and persist the model. + * + *

The fixture holds three tiny sentences inline, so the test needs no external data. + * A real treebank provides thousands of sentences; here each fixture sentence is + * repeated to give the trainer the same evidence many times, which lets the model + * memorize the fixture and makes every expected value exact.

+ */ +public class ConlluDependencyParserUsageTest { + + /** + * Joins the ten CoNLL-U columns of one word line with tabs. + * + * @param fields The column values; exactly ten are expected by the format. + * @return The joined word line. Never {@code null}. + */ + private static String line(String... fields) { + return String.join("\t", fields); + } + + /** + * The training fixture: three gold sentences in CoNLL-U form. The {@code HEAD} column + * is one-based with {@code 0} marking the root; the reader converts it to the + * zero-based convention of {@link DependencyGraph}. + */ + private static final String CONLLU = String.join("\n", + "# text = the dog barks", + line("1", "the", "the", "DET", "DT", "_", "2", "det", "_", "_"), + line("2", "dog", "dog", "NOUN", "NN", "_", "3", "nsubj", "_", "_"), + line("3", "barks", "bark", "VERB", "VBZ", "_", "0", "root", "_", "_"), + "", + "# text = dogs bark", + line("1", "dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "", + "# text = she eats fish", + line("1", "she", "she", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "eats", "eat", "VERB", "VBZ", "_", "0", "root", "_", "_"), + line("3", "fish", "fish", "NOUN", "NN", "_", "2", "obj", "_", "_"), + "") + "\n"; + + private static DependencyModel model; + private static DependencyParserME parser; + + /** + * Reads the fixture sentences through the CoNLL-U reader. + * + * @return One sample per fixture sentence, in file order. Never {@code null}. + * @throws IOException Thrown if reading the in-memory content fails. + */ + private static List readFixture() throws IOException { + final InputStreamFactory in = + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + final List samples = new ArrayList<>(); + try (ConlluDependencySampleStream stream = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + DependencySample sample; + while ((sample = stream.read()) != null) { + samples.add(sample); + } + } + return samples; + } + + /** + * Trains the parser once for all tests: read the fixture, repeat it for evidence, + * and hand the samples to the trainer. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final List fixture = readFixture(); + final List trainingSamples = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + trainingSamples.addAll(fixture); + } + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(trainingSamples), parameters); + parser = new DependencyParserME(model); + } + + @Test + void testReaderDeliversTheGoldAnnotation() throws IOException { + final List fixture = readFixture(); + assertEquals(3, fixture.size()); + final DependencySample first = fixture.get(0); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), first.getGraph()); + assertEquals("NOUN", first.getTags()[1]); + } + + @Test + void testParseAssignsHeadsAndRelations() { + // Parsing takes the tokens and their part-of-speech tags; the result names, for + // every token, its head token and the relation between the two. + final DependencyGraph parse = parser.parse( + new String[] {"the", "dog", "barks"}, new String[] {"DET", "NOUN", "VERB"}); + assertEquals(1, parse.headOf(0)); + assertEquals("det", parse.relationOf(0)); + assertEquals(2, parse.headOf(1)); + assertEquals("nsubj", parse.relationOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, parse.headOf(2)); + assertEquals("root", parse.relationOf(2)); + assertEquals(2, parse.root()); + } + + @Test + void testEvaluatorScoresTheParserAgainstGoldSamples() throws IOException { + // The evaluator parses each gold sentence and accumulates the two standard scores; + // on its own training fixture the memorizing model is exact on all eight tokens. + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(readFixture())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(8, evaluator.getWordCount()); + } + + @Test + void testPersistedModelParsesLikeTheOriginal(@TempDir Path dir) throws IOException { + // A trained model is saved to a file and loaded back like any other tool model; the + // reloaded parser must produce the exact same parse as the original. + final Path file = dir.resolve("en-depparse.bin"); + model.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRON", "VERB", "NOUN"})); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index b7fb44cedd..f0a1a40740 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -132,6 +132,41 @@ void testMalformedLineFailsLoud() { () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); } + @Test + void testSemanticallyInvalidAnnotationIsSkippedNotFatal() throws IOException { + // Structurally well-formed lines whose annotation cannot form a valid tree, here an + // out-of-range head and a rootless cycle, skip the sentence instead of failing, so + // one broken sentence cannot abort reading a large treebank. + final String content = String.join("\n", + line("1", "far", "far", "ADV", "RB", "_", "5", "advmod", "_", "_"), + line("2", "off", "off", "ADP", "RP", "_", "0", "root", "_", "_"), + "", + line("1", "loop", "loop", "NOUN", "NN", "_", "2", "dep", "_", "_"), + line("2", "back", "back", "ADV", "RB", "_", "1", "dep", "_", "_"), + "", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample onlyValid = samples.read(); + assertNotNull(onlyValid); + assertArrayEquals(new String[] {"Fine"}, onlyValid.getTokens()); + assertEquals(DependencyArc.ROOT_HEAD, onlyValid.getGraph().headOf(0)); + assertNull(samples.read()); + } + } + + @Test + void testEmptyContentYieldsNoSample() throws IOException { + final InputStreamFactory in = () -> new ByteArrayInputStream(new byte[0]); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + assertNull(samples.read()); + } + } + @Test void testValidation() { assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java index bd2d05f0ec..596ef4d2d4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java @@ -35,7 +35,7 @@ public final class ArcStandardOracle { private ArcStandardOracle() { - // static oracle, not meant to be instantiated + // This class only exposes static derivation methods and is never instantiated. } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java index 10dbcba958..8778071c77 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -37,7 +37,7 @@ final class FeedforwardContext { static final int LABEL_POSITIONS = 8; private FeedforwardContext() { - // static template only + // This class only exposes the static feature template and is never instantiated. } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index 0ada7c985a..aa5b64f622 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -27,10 +27,10 @@ * with a beam size above one. * *

With a beam, the decoder keeps the highest scoring transition sequences side by - * side, scored by summed log-probabilities, so one locally attractive but globally - * wrong transition no longer commits the whole parse. Every complete arc-standard - * derivation of a sentence has the same length, which keeps the summed scores - * comparable without length normalization.

+ * side, scored by summed log-probabilities, so a single locally attractive but globally + * wrong transition can still be recovered while the correct parse remains inside the + * beam. Every complete arc-standard derivation of a sentence has the same length, which + * keeps the summed scores comparable without length normalization.

* *

Inference is ordinary array arithmetic with no native runtime involved, so this * parser deploys exactly like the classical one while scoring configurations with diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index c676949ca9..deab7611d6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -51,7 +51,7 @@ public final class FeedforwardDependencyTrainer { private static final double ADAGRAD_EPSILON = 1e-6; private FeedforwardDependencyTrainer() { - // static trainer only + // This class only exposes static training methods and is never instantiated. } /** @@ -373,7 +373,9 @@ private double refineSentence(DependencySample sample, int[] oracle, } if (!goldSurvives) { if (goldChild == null) { - return -1.0; // the oracle transition was inapplicable; nothing to learn from + // The gold transition was not applicable in the gold configuration, so + // this sentence yields no update. + return -1.0; } survivors.add(goldChild); return updateFromCandidates(survivors); @@ -553,7 +555,9 @@ private static void seed(FeedforwardDependencyModel model, int seeded = 0; for (final Map.Entry entry : model.wordIds().entrySet()) { if (entry.getKey().startsWith("*")) { - continue; // the special symbols have no pretrained meaning + // The special unknown, padding, and root symbols have no pretrained + // counterpart, so they keep their random initialization. + continue; } final float[] vector = pretrained.apply(entry.getKey()); if (vector == null) { @@ -590,7 +594,8 @@ private static FeedforwardDependencyModel initialize(List corp labelIds.putIfAbsent(graph.relationOf(i), 0); } } - // the outcome space: shift plus both arc directions for every observed label + // The outcome space is the shift transition plus both arc directions for every + // relation label observed in the training data. transitionIds.putIfAbsent(Transition.SHIFT.encode(), 0); for (final String label : labelIds.keySet()) { transitionIds.putIfAbsent(Transition.leftArc(label).encode(), 0); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java new file mode 100644 index 0000000000..9cc6cc0b47 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the configuration mechanics of {@link ArcStandardState}: the start + * configuration, transition applicability at the boundaries, the bookkeeping of attached + * dependents, copy independence, and the fail-loud behavior of every accessor. + */ +public class ArcStandardStateTest { + + @Test + void testInitialConfiguration() { + final ArcStandardState state = new ArcStandardState(3); + assertEquals(ArcStandardState.ROOT, state.stack(0)); + assertEquals(ArcStandardState.NONE, state.stack(1)); + assertEquals(0, state.buffer(0)); + assertEquals(1, state.buffer(1)); + assertEquals(2, state.buffer(2)); + assertEquals(ArcStandardState.NONE, state.buffer(3)); + assertEquals(1, state.stackSize()); + assertEquals(3, state.bufferSize()); + assertFalse(state.isTerminal()); + } + + @Test + void testSingleTokenDerivationIsForced() { + // With one token the system permits exactly one derivation: shift the token, then + // attach it to the artificial root with a right arc. + final ArcStandardState state = new ArcStandardState(1); + assertTrue(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertFalse(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.SHIFT); + assertFalse(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertTrue(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + state.toGraph()); + } + + @Test + void testDependentBookkeepingDuringADerivation() { + // Derives "the dog barks" (the<-dog via det, dog<-barks via nsubj, barks<-root) and + // checks the partial-structure accessors after every attachment. + final ArcStandardState state = new ArcStandardState(3); + state.apply(Transition.SHIFT); + state.apply(Transition.SHIFT); + assertEquals(0, state.assignedDependents(1)); + assertNull(state.assignedRelation(0)); + + state.apply(Transition.leftArc("det")); + assertEquals(1, state.assignedDependents(1)); + assertEquals(0, state.leftmostDependent(1)); + assertEquals(0, state.rightmostDependent(1)); + assertEquals("det", state.assignedRelation(0)); + + state.apply(Transition.SHIFT); + state.apply(Transition.leftArc("nsubj")); + assertEquals(1, state.leftmostDependent(2)); + assertEquals("nsubj", state.assignedRelation(1)); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), state.toGraph()); + } + + @Test + void testInapplicableTransitionFailsLoud() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.leftArc("det"))); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.rightArc("root"))); + assertThrows(IllegalArgumentException.class, () -> state.apply(null)); + assertThrows(IllegalArgumentException.class, () -> state.canApply(null)); + } + + @Test + void testToGraphBeforeTerminalFailsLoud() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalStateException.class, state::toGraph); + state.apply(Transition.SHIFT); + assertThrows(IllegalStateException.class, state::toGraph); + } + + @Test + void testCopyIsIndependentOfTheOriginal() { + final ArcStandardState original = new ArcStandardState(2); + original.apply(Transition.SHIFT); + final ArcStandardState copy = original.copy(); + + copy.apply(Transition.SHIFT); + copy.apply(Transition.leftArc("nsubj")); + // The copy advanced by two transitions while the original still has one token + // buffered and one on the stack. + assertEquals(2, original.stackSize()); + assertEquals(1, original.bufferSize()); + assertEquals(0, original.assignedDependents(1)); + assertEquals(1, copy.assignedDependents(1)); + } + + @Test + void testAccessorValidation() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, () -> state.stack(-1)); + assertThrows(IllegalArgumentException.class, () -> state.buffer(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(2)); + assertThrows(IllegalArgumentException.class, () -> state.leftmostDependent(2)); + assertThrows(IllegalArgumentException.class, () -> state.rightmostDependent(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedRelation(2)); + } + + @Test + void testTokenCountMustBePositive() { + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(-1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java new file mode 100644 index 0000000000..f7015089e9 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the boundary behavior of both dependency parsers: empty and single-token input, + * non-projective sentences during training and parsing, and the file round trip of both + * model formats, which must reproduce the exact parses of the original models. + */ +public class DependencyParserEdgeCaseTest { + + private static DependencyModel maxentModel; + private static DependencyParserME maxentParser; + private static FeedforwardDependencyModel feedforwardModel; + private static FeedforwardDependencyParser feedforwardParser; + + /** + * Builds one gold sample from its parallel arrays. + * + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @param heads The zero-based head per token, {@code -1} for the root. + * @param relations The relation label per token. + * @return The assembled sample. Never {@code null}. + */ + private static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + /** + * Builds the projective training corpus: three tiny sentences, each repeated so both + * trainers see enough evidence to memorize them. + * + * @return The training samples. Never {@code null}. + */ + private static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + /** + * Builds a four-token sample whose gold arcs (2,0) and (3,1) cross, so the tree is + * non-projective and has no arc-standard derivation. + * + * @return The non-projective sample. Never {@code null}. + */ + private static DependencySample nonProjectiveSample() { + return sample(new String[] {"the", "dog", "barks", "today"}, + new String[] {"DT", "NN", "VBZ", "RB"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "nsubj", "root", "advmod"}); + } + + /** + * Trains one classical and one neural model on the shared corpus. The feedforward + * settings disable dropout and fix the seed, so the tiny network memorizes the corpus + * deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParsers() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + maxentModel = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters); + maxentParser = new DependencyParserME(maxentModel); + + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + feedforwardModel = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + feedforwardParser = new FeedforwardDependencyParser(feedforwardModel); + } + + @Test + void testEmptySentenceIsRejectedByBothParsers() { + assertThrows(IllegalArgumentException.class, + () -> maxentParser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> feedforwardParser.parse(new String[0], new String[0])); + // The transition system itself has no configuration for zero tokens either. + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + } + + @Test + void testSingleTokenSentenceAttachesToTheRoot() { + // A single token permits only the derivation shift then right-arc, so the head is + // forced to the artificial root and the model only chooses the relation label. + final DependencyGraph maxentParse = + maxentParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), maxentParse); + + final DependencyGraph feedforwardParse = + feedforwardParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + feedforwardParse); + } + + @Test + void testNonProjectiveSamplesAreSkippedDuringTraining() throws IOException { + // One non-projective sample joins the corpus; it cannot yield events, so training + // proceeds on the remaining samples and still memorizes the projective sentences. + final List mixed = new ArrayList<>(corpus()); + mixed.add(nonProjectiveSample()); + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + final DependencyModel model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(mixed), parameters); + final DependencyParserME parser = new DependencyParserME(model); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + parser.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"})); + } + + @Test + void testNonProjectiveGoldDecodesToAProjectiveTree() { + // The parser can only emit arc-standard derivations, so for a sentence whose gold + // tree is non-projective the prediction is necessarily a different, projective tree. + final DependencySample gold = nonProjectiveSample(); + final DependencyGraph parsed = maxentParser.parse(gold.getTokens(), gold.getTags()); + assertNotEquals(gold.getGraph(), parsed); + assertEquals(0, crossingArcCount(parsed)); + // The unseen final token becomes the root and the verb attaches under it; the + // familiar determiner and subject arcs survive from the training evidence. + assertEquals(DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"det", "nsubj", "nsubj", "root"}), parsed); + } + + @Test + void testFeedforwardTrainingFailsLoudWithoutProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(8, 8, 1, 32, 0.05, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings)); + assertEquals("no trainable examples in the samples", e.getMessage()); + } + + @Test + void testRefinementFailsLoudWithoutProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(feedforwardModel, + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings, 2)); + assertEquals("no trainable samples for refinement", e.getMessage()); + } + + @Test + void testMaxentModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse.bin"); + maxentModel.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + for (final DependencySample sample : corpus()) { + assertEquals(maxentParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags())); + } + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + reloaded.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + @Test + void testFeedforwardModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse-ff.bin"); + try (OutputStream out = Files.newOutputStream(file)) { + feedforwardModel.serialize(out); + } + final FeedforwardDependencyParser reloaded = + new FeedforwardDependencyParser(FeedforwardDependencyModel.load(file)); + for (final DependencySample sample : corpus()) { + assertEquals(feedforwardParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags())); + } + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"})); + } + + /** + * Counts the pairs of crossing arcs in a graph, treating the root arc as spanning + * from a virtual position left of the sentence to its dependent. A projective tree + * has zero crossing pairs. + * + * @param graph The graph to inspect. Must not be {@code null}. + * @return The number of crossing arc pairs. + * @throws IllegalArgumentException Thrown if {@code graph} is {@code null}. + */ + private static int crossingArcCount(DependencyGraph graph) { + if (graph == null) { + throw new IllegalArgumentException("graph must not be null"); + } + int crossings = 0; + for (int i = 0; i < graph.size(); i++) { + for (int j = i + 1; j < graph.size(); j++) { + final int iLow = Math.min(i, graph.headOf(i)); + final int iHigh = Math.max(i, graph.headOf(i)); + final int jLow = Math.min(j, graph.headOf(j)); + final int jHigh = Math.max(j, graph.headOf(j)); + if ((iLow < jLow && jLow < iHigh && iHigh < jHigh) + || (jLow < iLow && iLow < jHigh && jHigh < iHigh)) { + crossings++; + } + } + } + return crossings; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java new file mode 100644 index 0000000000..a46866855c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the outcome encoding of {@link Transition}: every transition must render to a + * unique outcome string and decode back to an equal transition, and every malformed + * outcome must be rejected loudly. + */ +public class TransitionTest { + + @Test + void testEncodeRendersTheOutcomeStrings() { + assertEquals("SHIFT", Transition.SHIFT.encode()); + assertEquals("LEFT_ARC:nsubj", Transition.leftArc("nsubj").encode()); + assertEquals("RIGHT_ARC:obj", Transition.rightArc("obj").encode()); + } + + @Test + void testDecodeRestoresEncodedTransitions() { + assertSame(Transition.SHIFT, Transition.decode("SHIFT")); + assertEquals(Transition.leftArc("nsubj"), Transition.decode("LEFT_ARC:nsubj")); + assertEquals(Transition.rightArc("obj"), Transition.decode("RIGHT_ARC:obj")); + } + + @Test + void testLabelContainingTheSeparatorRoundTrips() { + // Only the first separator splits type from label, so a label containing the + // separator character itself survives the round trip unchanged. + final Transition transition = Transition.leftArc("nmod:poss"); + assertEquals("LEFT_ARC:nmod:poss", transition.encode()); + assertEquals(transition, Transition.decode(transition.encode())); + } + + @Test + void testDecodeRejectsMalformedOutcomes() { + assertThrows(IllegalArgumentException.class, () -> Transition.decode(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN")); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN:det")); + // A labeled shift is contradictory and must be rejected by the record invariant. + assertThrows(IllegalArgumentException.class, () -> Transition.decode("SHIFT:det")); + } + + @Test + void testConstructorValidation() { + assertThrows(IllegalArgumentException.class, () -> new Transition(null, "det")); + assertThrows(IllegalArgumentException.class, + () -> new Transition(Transition.Type.SHIFT, "det")); + assertThrows(IllegalArgumentException.class, () -> Transition.leftArc(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.rightArc(" ")); + } +} From a70ae3840806d19078b57fdc9ac5b000d3371c63 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:35:47 -0400 Subject: [PATCH 71/92] depparse: Document treebank acquisition for the gated evaluation with a download helper --- .../dev/README-ud-treebanks.md | 48 ++++++++++++++ .../dev/download-ud-treebank.sh | 66 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 opennlp-core/opennlp-formats/dev/README-ud-treebanks.md create mode 100755 opennlp-core/opennlp-formats/dev/download-ud-treebank.sh diff --git a/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md new file mode 100644 index 0000000000..3052d88553 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md @@ -0,0 +1,48 @@ + + +# Universal Dependencies treebanks for the dependency parser evaluation + +The dependency parser's unit tests are fully self-contained, but its accuracy evaluation runs against a real Universal Dependencies treebank that the user downloads. Apache OpenNLP bundles no treebank data, distributes none, and ships no models trained on it; the treebanks are used to reproduce accuracy numbers on your own machine. + +## Getting a treebank + +Every UD treebank lives in its own repository under `github.com/UniversalDependencies`, with its splits named `-ud-train.conllu`, `-dev`, and `-test`. The helper next to this file clones one shallowly and lays the splits out under the names the evaluation expects: + +``` +./download-ud-treebank.sh UD_English-EWT /tmp/ud-ewt +``` + +produces `/tmp/ud-ewt/train.conllu` and `/tmp/ud-ewt/test.conllu`. Any treebank that publishes both splits works the same way. + +## Running the gated evaluation + +`ConlluDependencyParserEvalTest` is disabled unless the `opennlp.depparse.ud.dir` system property points at a directory containing `train.conllu` and `test.conllu`: + +``` +./mvnw -pl opennlp-core/opennlp-formats test \ + -Dtest=ConlluDependencyParserEvalTest \ + -Dopennlp.depparse.ud.dir=/tmp/ud-ewt +``` + +Without the property the test reports as skipped, which is why a plain build never needs network access or external data. + +Two properties of the parser worth knowing when reading the numbers: multiword-token sentences are kept because the CoNLL-U reader recovers their dependency rows, and non-projective training sentences are skipped, since the arc-standard transition system cannot derive them; the skip count is inherent to the algorithm, not data loss in the reader. + +## Licensing + +Each treebank carries its own license, stated in its repository README, and downloading one means accepting those terms yourself. The annotations of `UD_English-EWT`, for example, are licensed under CC BY-SA 4.0. The project's handling: treebanks are benchmark inputs on the user's machine only; no treebank data enters the source tree or any release artifact, and the project publishes no models trained on share-alike data. If you train and distribute your own model from a treebank, checking that treebank's terms is your responsibility. diff --git a/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh new file mode 100755 index 0000000000..f229b57a31 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fetches one Universal Dependencies treebank and lays its splits out the way the +# gated dependency-parser evaluation expects: /train.conllu and +# /test.conllu. See README-ud-treebanks.md in this directory for the +# evaluation command and the licensing notes; each treebank carries its own license, +# which you accept by downloading it. Nothing is bundled with Apache OpenNLP. + +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 + echo "" >&2 + echo " treebank-repository a repository name under github.com/UniversalDependencies," >&2 + echo " for example UD_English-EWT" >&2 + echo " target-dir where train.conllu and test.conllu are placed" >&2 + exit 2 +} + +[ $# -ne 2 ] && usage +treebank="$1" +target="$2" + +# Clone shallowly into a temporary directory; only the .conllu files are kept. +clone="$(mktemp -d)" +trap 'rm -rf "${clone}"' EXIT +echo "cloning ${treebank}" +git clone --quiet --depth 1 "https://github.com/UniversalDependencies/${treebank}.git" \ + "${clone}/${treebank}" + +mkdir -p "${target}" +for split in train test; do + # UD names its files -ud-.conllu; the code prefix varies per + # treebank, so match on the stable -ud- suffix. + found="" + for f in "${clone}/${treebank}/"*"-ud-${split}.conllu"; do + [ -e "$f" ] && found="$f" && break + done + if [ -z "${found}" ]; then + echo "no *-ud-${split}.conllu in ${treebank}; the treebank may not publish" >&2 + echo "that split (some hide test data or ship dev only)" >&2 + exit 1 + fi + cp "${found}" "${target}/${split}.conllu" + echo "wrote ${target}/${split}.conllu ($(wc -l < "${target}/${split}.conllu") lines)" +done + +echo "" +echo "run the gated evaluation with:" +echo " ./mvnw -pl opennlp-core/opennlp-formats test \\" +echo " -Dtest=ConlluDependencyParserEvalTest \\" +echo " -Dopennlp.depparse.ud.dir=${target}" From e48a5c948f1423b8abfc30445a434f79216fe9e3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 22:06:55 -0400 Subject: [PATCH 72/92] depparse: Refine into a new model, fail loud on unknown transitions, and read blanks and case through the project seams --- .../conllu/ConlluDependencySampleStream.java | 26 +++++++- .../ConlluDependencySampleStreamTest.java | 25 ++++++++ .../depparse/FeedforwardDependencyModel.java | 57 ++++++++++++++--- .../FeedforwardDependencyTrainer.java | 37 ++++++++--- .../FeedforwardDependencyParserTest.java | 61 +++++++++++++++++++ 5 files changed, 188 insertions(+), 18 deletions(-) diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 22f9de9018..9b5154691b 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -31,6 +31,7 @@ import opennlp.tools.depparse.DependencySample; import opennlp.tools.util.InputStreamFactory; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; /** * Reads {@link DependencySample samples} directly from CoNLL-U content, mapping the @@ -112,7 +113,7 @@ private List nextSentence() throws IOException { final List words = new ArrayList<>(); String line; while ((line = reader.readLine()) != null) { - if (line.isBlank()) { + if (isBlank(line)) { if (!words.isEmpty()) { return words; } @@ -133,6 +134,29 @@ private List nextSentence() throws IOException { return words; } + /** + * Determines whether a line separates two sentences, that is whether it is empty or + * consists entirely of whitespace. + * + *

Blankness is decided with {@link StringUtil#isWhitespace(int)} rather than + * {@link String#isBlank()}, because OpenNLP counts the Unicode {@code Zs} category as + * whitespace while the JDK predicate does not: a separator line carrying a stray + * no-break space is still a separator, not a malformed word line.

+ * + * @param line The line to inspect. Must not be {@code null}. + * @return {@code true} if the line is blank, {@code false} otherwise. + */ + private static boolean isBlank(String line) { + for (int i = 0; i < line.length(); ) { + final int codePoint = line.codePointAt(i); + if (!StringUtil.isWhitespace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + /** * Converts one sentence, or returns {@code null} when its annotation is unusable. */ diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index f0a1a40740..01dbfaafb9 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -158,6 +158,31 @@ void testSemanticallyInvalidAnnotationIsSkippedNotFatal() throws IOException { } } + @Test + void testSeparatorLineOfNonBreakingSpaceSeparatesSentences() throws IOException { + // A separator line carrying a stray no-break space is still a separator: OpenNLP + // counts U+00A0 as whitespace, so such a line must not reach the word-line parser + // and abort the stream. + final String content = String.join("\n", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "\u00A0", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"Dogs", "bark"}, first.getTokens()); + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"Fine"}, second.getTokens()); + assertNull(samples.read()); + } + } + @Test void testEmptyContentYieldsNoSample() throws IOException { final InputStreamFactory in = () -> new ByteArrayInputStream(new byte[0]); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index e1457efcab..d19662373f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -27,9 +27,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; -import java.util.Locale; import java.util.Map; +import opennlp.tools.util.StringUtil; + /** * The weights of the feedforward transition parser: embeddings for words, tags, and arc * labels, one hidden layer with cube activation, and a transition output layer, stored @@ -38,8 +39,14 @@ *

This is the pure-Java neural tier: the network is executed with ordinary array * arithmetic, so parsing needs no native runtime, and the same class scores * configurations for training and decoding. Unknown words fall back to a learned - * unknown symbol; words are matched case-insensitively. Instances are immutable and - * safe to share between threads.

+ * unknown symbol; words are matched case-insensitively after + * {@link #normalize(String) normalization}.

+ * + *

An instance is immutable and safe to share between threads once it has been handed + * to a caller. {@link FeedforwardDependencyTrainer} fills the weights while building a + * model and before that model escapes, and + * {@link FeedforwardDependencyTrainer#refine refine} trains a copy rather than the model + * it is given, so no model a caller holds ever changes underneath it.

* * @see FeedforwardDependencyParser * @see FeedforwardDependencyTrainer @@ -68,9 +75,9 @@ public class FeedforwardDependencyModel { Map labelIds, String[] transitions, int embeddingSize, float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias, float[][] outputWeights, float[] outputBias) { - this.wordIds = wordIds; - this.tagIds = tagIds; - this.labelIds = labelIds; + this.wordIds = Map.copyOf(wordIds); + this.tagIds = Map.copyOf(tagIds); + this.labelIds = Map.copyOf(labelIds); this.transitions = transitions; this.embeddingSize = embeddingSize; this.embeddings = embeddings; @@ -142,12 +149,22 @@ public String[] transitions() { return transitions.clone(); } - /** Lowercases a word symbol; special symbols and absences pass through. */ + /** + * Lowercases a word symbol; special symbols and absences pass through. + * + *

Case is mapped with {@link StringUtil#toLowerCase(CharSequence)}, which maps each + * code point through UnicodeData, so no word grows a character on the way into the + * vocabulary and every OpenNLP component derives the same key for the same word.

+ * + * @param word The word to normalize. May be {@code null}. + * @return The vocabulary key of {@code word}, or {@code null} if {@code word} is + * {@code null}. + */ static String normalize(String word) { if (word == null) { return null; } - return word.startsWith("*") ? word : word.toLowerCase(Locale.ROOT); + return word.startsWith("*") ? word : StringUtil.toLowerCase(word); } private static int lookup(Map ids, String symbol) { @@ -289,6 +306,30 @@ private static float[] readVector(DataInputStream data) throws IOException { return vector; } + /** + * Creates an independent copy of this model: the weights are deep-copied, and the + * vocabularies and the transition inventory are shared because they are immutable. + * + *

This lets a training pass update the copy without ever writing to a model a + * caller already holds, which is what keeps the immutability this class documents + * true.

+ * + * @return A copy of this model sharing no mutable state with it. Never {@code null}. + */ + FeedforwardDependencyModel copy() { + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions.clone(), + embeddingSize, copyOf(embeddings), copyOf(hiddenWeights), hiddenBias.clone(), + copyOf(outputWeights), outputBias.clone()); + } + + private static float[][] copyOf(float[][] matrix) { + final float[][] copy = new float[matrix.length][]; + for (int r = 0; r < matrix.length; r++) { + copy[r] = matrix[r].clone(); + } + return copy; + } + Map wordIds() { return wordIds; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index deab7611d6..d1ec3e840b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -169,19 +169,29 @@ public static FeedforwardDependencyModel train(ObjectStream sa * beamed parser scores them, summed log-probabilities, so training optimizes the * quantity decoding uses. * - *

The model is updated in place with per-sentence AdaGrad steps and no dropout; - * {@link Settings#epochs()} counts the refinement passes. Refinement is deterministic - * for a fixed {@link Settings#seed()}. Parse afterwards with the same beam size.

+ *

The refined weights are a copy: {@code model} itself is never written to, so a + * model already being parsed with, possibly by several threads, keeps behaving exactly + * as before while a refined successor is trained from it. The copy is updated with + * per-sentence AdaGrad steps and no dropout; {@link Settings#epochs()} counts the + * refinement passes. Refinement is deterministic for a fixed {@link Settings#seed()}. + * Parse afterwards with the same beam size.

* - * @param model The locally trained model to refine. Must not be {@code null}. + *

The transition inventory comes from {@code model} and is not extended, because + * its size is the width of the trained output layer. A refinement corpus using a + * relation label the original training set lacked is therefore rejected rather than + * silently ignored.

+ * + * @param model The locally trained model to refine. Left untouched. Must not be + * {@code null}. * @param samples The training samples. Must not be {@code null}. * @param settings The hyperparameters; {@code epochs}, {@code learningRate}, * {@code l2}, and {@code seed} apply. Must not be {@code null}. * @param beamSize The beam width to track the gold derivation in. Must be at least 2. - * @return The same model instance, refined. Never {@code null}. + * @return A new refined model, distinct from {@code model}. Never {@code null}. * @throws IOException Thrown if reading the samples fails. * @throws IllegalArgumentException Thrown if a parameter is {@code null}, - * {@code beamSize} is below 2, or no trainable sample can be derived. + * {@code beamSize} is below 2, no trainable sample can be derived, or a sample + * requires a transition {@code model} does not know. */ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model, ObjectStream samples, Settings settings, int beamSize) @@ -215,7 +225,15 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model } final int[] encoded = new int[oracle.size()]; for (int i = 0; i < encoded.length; i++) { - encoded[i] = transitionIds.get(oracle.get(i).encode()); + final String outcome = oracle.get(i).encode(); + final Integer id = transitionIds.get(outcome); + if (id == null) { + // The outcome space was fixed by the original training set, so a relation + // label it never saw has no output unit to push probability onto. + throw new IllegalArgumentException( + "unknown transition in the refinement samples: " + outcome); + } + encoded[i] = id; } trainable.add(s); oracles.add(encoded); @@ -224,7 +242,8 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model throw new IllegalArgumentException("no trainable samples for refinement"); } - final GlobalOptimizer optimizer = new GlobalOptimizer(model, settings); + final FeedforwardDependencyModel refined = model.copy(); + final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings); final Random random = new Random(settings.seed()); final int[] order = new int[trainable.size()]; for (int i = 0; i < order.length; i++) { @@ -246,7 +265,7 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch, loss / Math.max(updates, 1), updates, System.currentTimeMillis() - epochStart); } - return model; + return refined; } /** One candidate path in the refinement beam: the parent link forms the history. */ diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index a4a8c347d8..87597ed5c0 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -21,14 +21,20 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.StringUtil; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -142,6 +148,61 @@ void testRefinementKeepsToyPerformance() throws IOException { assertEquals(1.0d, evaluator.getLas()); } + @Test + void testRefineWithAnUnknownRelationFailsLoud() throws IOException { + // A refinement corpus may carry a relation label the original training set never + // used; the transition inventory is fixed at training time, so refinement cannot + // score it and must say which transition it does not know. + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final List unseenRelation = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "dislocated", "root"})); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(model, + ObjectStreamUtils.createObjectStream(unseenRelation), settings, 2)); + assertEquals("unknown transition in the refinement samples: LEFT_ARC:dislocated", + e.getMessage()); + } + + @Test + void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = local.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + final double[] before = local.score(features); + + final FeedforwardDependencyTrainer.Settings refineSettings = + new FeedforwardDependencyTrainer.Settings(16, 32, 2, 32, 0.01, 0.0, 0.0, 1, 17L); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + // refinement produces a distinct model, so a model already shared between threads + // cannot change underneath them + assertNotSame(local, refined); + assertArrayEquals(before, local.score(features)); + // and the returned model really carries the refinement + assertFalse(Arrays.equals(before, refined.score(features))); + } + + @Test + void testNormalizeUsesTheUnicodeDataCaseMapping() { + // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's + // String.toLowerCase would render this word as "i" + COMBINING DOT ABOVE instead. + assertEquals(StringUtil.toLowerCase("\u0130STANBUL"), + FeedforwardDependencyModel.normalize("\u0130STANBUL")); + assertEquals("istanbul", FeedforwardDependencyModel.normalize("\u0130STANBUL")); + // special symbols still pass through untouched + assertEquals(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.normalize(FeedforwardDependencyModel.UNKNOWN)); + assertNull(FeedforwardDependencyModel.normalize(null)); + } + @Test void testRefineValidation() { final FeedforwardDependencyTrainer.Settings settings = From e83f03cb5839ac7284635f4ec26095afa8e0e05d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 00:30:33 -0400 Subject: [PATCH 73/92] depparse: Restore the final-sigma rule in vocabulary normalization and make model bytes reproducible Routing normalization through the plain per-code-point mapping broke case-insensitive vocabulary matching for Greek: treebank-derived keys spell a word-final sigma as U+03C2, and an uppercase surface form mapped per code point ends in the medial U+03C3 instead, missing the vocabulary. Normalization now applies the Final_Sigma condition of the Unicode SpecialCasing file, restricted to a single token, on top of the per-code-point mapping, and returns already-lowercase words unchanged without allocating, which is the common case on the parse hot path. Serialized vocabularies are written in ascending id order because the iteration order of the immutable maps is salted per JVM launch, so serializing the same model now produces the same bytes on every run. Transition and dependency-graph relation labels judge blankness under the project whitespace definition, and the copy javadoc no longer calls the cloned transition array immutable. --- .../tools/depparse/DependencyGraph.java | 20 ++++- .../depparse/FeedforwardDependencyModel.java | 63 ++++++++++++-- .../opennlp/tools/depparse/Transition.java | 20 ++++- .../FeedforwardDependencyParserTest.java | 87 +++++++++++++++++++ 4 files changed, 181 insertions(+), 9 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index e95234dcc8..64921d77f3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -22,6 +22,8 @@ import java.util.Collections; import java.util.List; +import opennlp.tools.util.StringUtil; + /** * An immutable dependency tree over one sentence: for every token, the index of its head * and the label of the relation to that head. @@ -77,7 +79,7 @@ public static DependencyGraph of(int[] heads, String[] relations) { } else if (heads[i] == i) { throw new IllegalArgumentException("token " + i + " must not head itself"); } - if (relations[i] == null || relations[i].isBlank()) { + if (relations[i] == null || blank(relations[i])) { throw new IllegalArgumentException("relation of token " + i + " must not be blank"); } } @@ -87,6 +89,22 @@ public static DependencyGraph of(int[] heads, String[] relations) { return new DependencyGraph(heads.clone(), relations.clone()); } + /** + * Reports whether a relation label is blank under the project whitespace + * definition, which unlike the JDK's includes no-break spaces, so a label spelled + * entirely from them cannot pass as a relation. + */ + private static boolean blank(String value) { + for (int i = 0; i < value.length(); ) { + final int cp = value.codePointAt(i); + if (!StringUtil.isWhitespace(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + /** * @return The number of tokens the graph spans. */ diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index d19662373f..cf2d015aaf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -26,7 +26,9 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import opennlp.tools.util.StringUtil; @@ -56,6 +58,12 @@ public class FeedforwardDependencyModel { private static final String MAGIC = "ONLP-FFDP-1"; + /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point with a contextual lowering. */ + private static final int GREEK_CAPITAL_SIGMA = 0x03A3; + + /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the capital. */ + private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2'; + static final String UNKNOWN = "*UNK*"; static final String ABSENT = "*NULL*"; @@ -152,9 +160,15 @@ public String[] transitions() { /** * Lowercases a word symbol; special symbols and absences pass through. * - *

Case is mapped with {@link StringUtil#toLowerCase(CharSequence)}, which maps each - * code point through UnicodeData, so no word grows a character on the way into the - * vocabulary and every OpenNLP component derives the same key for the same word.

+ *

Case is mapped per code point through UnicodeData, the same mapping as + * {@link StringUtil#toLowerCase(CharSequence)}, with one contextual rule on top: a + * Greek capital sigma preceded by a letter and not followed by one lowercases to + * the final form U+03C2, the Final_Sigma condition of the Unicode SpecialCasing + * file restricted to a single token. Natural lowercase Greek text, and with it + * every vocabulary key derived from a treebank, spells a word-final sigma that + * way, so without the rule an uppercase Greek word would normalize to a spelling + * the vocabulary never contains. A word that is already lowercase, the common + * case at parse time, is returned unchanged without allocating.

* * @param word The word to normalize. May be {@code null}. * @return The vocabulary key of {@code word}, or {@code null} if {@code word} is @@ -164,7 +178,36 @@ static String normalize(String word) { if (word == null) { return null; } - return word.startsWith("*") ? word : StringUtil.toLowerCase(word); + if (word.startsWith("*")) { + return word; + } + int i = 0; + while (i < word.length()) { + final int cp = word.codePointAt(i); + if (Character.toLowerCase(cp) != cp) { + break; + } + i += Character.charCount(cp); + } + if (i == word.length()) { + return word; + } + final StringBuilder lowered = new StringBuilder(word.length()); + lowered.append(word, 0, i); + while (i < word.length()) { + final int cp = word.codePointAt(i); + final int width = Character.charCount(cp); + if (cp == GREEK_CAPITAL_SIGMA && i > 0 + && Character.isLetter(word.codePointBefore(i)) + && (i + width >= word.length() + || !Character.isLetter(word.codePointAt(i + width)))) { + lowered.append(GREEK_SMALL_FINAL_SIGMA); + } else { + lowered.appendCodePoint(Character.toLowerCase(cp)); + } + i += width; + } + return lowered.toString(); } private static int lookup(Map ids, String symbol) { @@ -251,7 +294,12 @@ public static FeedforwardDependencyModel load(Path path) throws IOException { private static void writeVocabulary(DataOutputStream data, Map ids) throws IOException { data.writeInt(ids.size()); - for (final Map.Entry entry : ids.entrySet()) { + // Entries are written in ascending id order: the iteration order of the immutable + // maps is salted per JVM launch, and serializing the same model must produce the + // same bytes on every run. + final List> entries = new ArrayList<>(ids.entrySet()); + entries.sort(Map.Entry.comparingByValue()); + for (final Map.Entry entry : entries) { data.writeUTF(entry.getKey()); data.writeInt(entry.getValue()); } @@ -307,8 +355,9 @@ private static float[] readVector(DataInputStream data) throws IOException { } /** - * Creates an independent copy of this model: the weights are deep-copied, and the - * vocabularies and the transition inventory are shared because they are immutable. + * Creates an independent copy of this model: the weights and the transition + * inventory array are deep-copied, and the vocabularies are shared because their + * maps are immutable. * *

This lets a training pass update the copy without ever writing to a model a * caller already holds, which is what keeps the immutability this class documents diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java index 0f9977e675..2cfa6f9282 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -17,6 +17,8 @@ package opennlp.tools.depparse; +import opennlp.tools.util.StringUtil; + /** * One action of the arc-standard transition system: shift the next buffer token onto the * stack, or attach one of the two topmost stack tokens to the other under a relation label. @@ -62,11 +64,27 @@ public enum Type { if (label != null) { throw new IllegalArgumentException("a shift must not carry a label: " + label); } - } else if (label == null || label.isBlank()) { + } else if (label == null || blank(label)) { throw new IllegalArgumentException("an arc transition needs a relation label"); } } + /** + * Reports whether a label is blank under the project whitespace definition, which + * unlike the JDK's includes no-break spaces, so a label spelled entirely from them + * cannot pass as a relation. + */ + private static boolean blank(String value) { + for (int i = 0; i < value.length(); ) { + final int cp = value.codePointAt(i); + if (!StringUtil.isWhitespace(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + /** * Creates a left-arc transition. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 87597ed5c0..2d239df018 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -35,7 +36,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests the pure-Java neural tier end to end: training on a tiny corpus must let the @@ -190,6 +193,90 @@ void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws IOExceptio assertFalse(Arrays.equals(before, refined.score(features))); } + /** + * Pins the one contextual rule layered over the per-code-point mapping: a Greek + * capital sigma preceded by a letter and not followed by one lowercases to the + * final form U+03C2, the way natural lowercase Greek spells it and the way every + * treebank-derived vocabulary key spells it, so an uppercase Greek word normalizes + * to a key the vocabulary can actually contain. The plain per-code-point mapping + * would produce the medial sigma there and miss the vocabulary. + */ + @Test + void testNormalizeAppliesTheFinalSigmaRule() { + // ODOS, road, all caps: the trailing sigma position lowers to U+03C2 + assertEquals("\u03BF\u03B4\u03BF\u03C2", + FeedforwardDependencyModel.normalize("\u039F\u0394\u039F\u03A3")); + // SOFIA: the word-initial sigma is not final and lowers to the medial U+03C3 + assertEquals("\u03C3\u03BF\u03C6\u03B9\u03B1", + FeedforwardDependencyModel.normalize("\u03A3\u039F\u03A6\u0399\u0391")); + // a lone capital sigma has no preceding letter, so the rule does not fire + assertEquals("\u03C3", FeedforwardDependencyModel.normalize("\u03A3")); + } + + /** + * Pins the allocation-free fast path: a word the mapping leaves unchanged, the + * overwhelming majority of parse-time input, is returned as the same instance + * rather than a fresh copy built on every lookup of the scoring loop. + */ + @Test + void testNormalizeReturnsTheSameInstanceForLowercaseWords() { + final String plain = "barks"; + assertSame(plain, FeedforwardDependencyModel.normalize(plain)); + // lowercase Greek with its native final sigma is already normalized + final String greek = "\u03BF\u03B4\u03BF\u03C2"; + assertSame(greek, FeedforwardDependencyModel.normalize(greek)); + } + + /** + * Pins the refinement contract for the sample kind the unknown-transition check + * never sees: a non-projective gold graph has no arc-standard derivation and is + * skipped before the transition inventory is consulted, so an unknown relation + * riding on it must not trigger the unknown-transition failure, and refinement + * proceeds on the remaining projective samples. + */ + @Test + void testNonProjectiveSampleWithUnknownRelationIsSkippedNotFatal() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + // heads {2, 3, -1, 2}: the arcs from 2 to 0 and from 3 to 1 cross, so the graph + // is non-projective, and "dislocated" is a relation the model was never trained on + final List mixed = List.of( + sample(new String[] {"a", "b", "c", "d"}, new String[] {"DT", "NN", "VBZ", "NN"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "dislocated", "root", "obj"}), + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"})); + + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + model, ObjectStreamUtils.createObjectStream(mixed), settings, 2); + assertNotSame(model, refined); + } + + /** + * Pins the serialized vocabulary order: entries are written in ascending id order, + * not in the iteration order of the underlying immutable maps, which the JDK salts + * per launch, so serializing the same model produces the same bytes on every run. + */ + @Test + void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + try (DataInputStream data = new DataInputStream( + new ByteArrayInputStream(out.toByteArray()))) { + data.readUTF(); + for (int vocab = 0; vocab < 3; vocab++) { + final int size = data.readInt(); + int previous = Integer.MIN_VALUE; + for (int entry = 0; entry < size; entry++) { + data.readUTF(); + final int id = data.readInt(); + assertTrue(id > previous, + "vocabulary " + vocab + " must be written in ascending id order"); + previous = id; + } + } + } + } + @Test void testNormalizeUsesTheUnicodeDataCaseMapping() { // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's From c8584a2652f848c9b20bb4b709b8154b2afdf335 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 06:48:34 -0400 Subject: [PATCH 74/92] depparse: Cache hidden-layer contributions per feature pair on frozen models The scorer re-derived every feature's hidden-layer contribution from its embedding on every configuration, although for a frozen model the contribution of a (template position, embedding row) pair is a fixed vector. Parsers now turn on a bounded lazy cache that computes each pair's vector once on first sight and adds it thereafter, the adaptive form of the precomputation described for this architecture by Chen and Manning (2014): tag and label rows are fully cached within a document or two and word rows follow their frequency. Measured on realistic dimensions (20k words, embedding 50, hidden 400, 77 transitions): 5,462 to 71,954 scored states per second, 13.2x. Training and refinement work on uncached copies, copies never carry a cache, concurrent readers are safe by idempotent fill, and a test pins cached-versus-direct agreement to float rounding with identical winning transitions. --- .../depparse/FeedforwardDependencyModel.java | 121 +++++++++++++++++- .../depparse/FeedforwardDependencyParser.java | 3 + .../FeedforwardDependencyParserTest.java | 36 ++++++ 3 files changed, 153 insertions(+), 7 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index cf2d015aaf..ddd906a880 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -30,6 +30,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; import opennlp.tools.util.StringUtil; @@ -67,6 +69,9 @@ public class FeedforwardDependencyModel { static final String UNKNOWN = "*UNK*"; static final String ABSENT = "*NULL*"; + /** The lazy scoring cache; {@code null} until {@link #enableScoringCache()}. */ + private volatile ContributionCache cache; + private final Map wordIds; private final Map tagIds; private final Map labelIds; @@ -107,16 +112,31 @@ public double[] score(int[] features) { final int hidden = hiddenBias.length; final double[] h = new double[hidden]; for (int j = 0; j < hidden; j++) { - final float[] row = hiddenWeights[j]; - double sum = hiddenBias[j]; - for (int f = 0; f < features.length; f++) { - final float[] embedding = embeddings[features[f]]; + h[j] = hiddenBias[j]; + } + final ContributionCache cache = this.cache; + for (int f = 0; f < features.length; f++) { + final int row = features[f]; + final float[] contribution = cache == null ? null : cache.contribution(this, f, row); + if (contribution != null) { + for (int j = 0; j < hidden; j++) { + h[j] += contribution[j]; + } + } else { + final float[] embedding = embeddings[row]; final int offset = f * embeddingSize; - for (int d = 0; d < embeddingSize; d++) { - sum += row[offset + d] * embedding[d]; + for (int j = 0; j < hidden; j++) { + final float[] weights = hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < embeddingSize; d++) { + sum += weights[offset + d] * embedding[d]; + } + h[j] += sum; } } - h[j] = sum * sum * sum; + } + for (int j = 0; j < hidden; j++) { + h[j] = h[j] * h[j] * h[j]; } final double[] scores = new double[transitions.length]; for (int o = 0; o < scores.length; o++) { @@ -130,6 +150,93 @@ public double[] score(int[] features) { return scores; } + /** + * Turns on the scoring cache: the hidden-layer contribution of a (template + * position, embedding row) pair is a fixed vector for a frozen model, so it is + * computed once on first sight and afterwards added instead of being re-derived + * from the embedding on every configuration. Tag and label rows, whose inventories + * are small, are fully cached within a document or two; word rows follow their + * frequency, which is the adaptive form of the precomputation described for this + * architecture by Chen and Manning (2014). + * + *

Cached contributions are rounded to floats once, so scores may differ from the + * uncached path in the last bits; transition decisions are unaffected at any + * realistic margin. The cache is bounded, safe for concurrent readers, and only + * valid on a model whose weights no longer change: training and refinement work on + * uncached copies, and {@link #copy()} never carries a cache over.

+ */ + void enableScoringCache() { + if (cache == null) { + cache = new ContributionCache(2 * FeedforwardContext.POSITIONS + + FeedforwardContext.LABEL_POSITIONS, embeddings.length); + } + } + + /** + * The bounded lazy contribution cache behind {@link #enableScoringCache()}: one + * slot per (template position, embedding row) pair, filled on first use. Filling is + * idempotent, so concurrent readers may compute a contribution twice but never see + * a partial one, and a shared budget bounds the total memory; pairs beyond the + * budget simply keep the direct path. + */ + private static final class ContributionCache { + + /** The most (position, row) pairs the cache will hold. At a hidden size of 400 + * this bounds the cache near 100 MB; typical models stay far below the cap + * because tag and label inventories are small and word usage is Zipf-shaped. */ + private static final int MAX_PAIRS = 65536; + + private final AtomicReferenceArray[] byPosition; + private final AtomicInteger remaining = new AtomicInteger(MAX_PAIRS); + + @SuppressWarnings("unchecked") + private ContributionCache(int positions, int rows) { + byPosition = new AtomicReferenceArray[positions]; + for (int f = 0; f < positions; f++) { + byPosition[f] = new AtomicReferenceArray<>(rows); + } + } + + /** + * Returns the cached hidden-layer contribution of one pair, computing and + * publishing it on first sight while the budget lasts. + * + * @param model The frozen model the contributions derive from. + * @param position The template position. + * @param row The embedding row at that position. + * @return The contribution vector, or {@code null} when the budget is spent and + * the pair is not cached. + */ + private float[] contribution(FeedforwardDependencyModel model, int position, int row) { + final AtomicReferenceArray slots = byPosition[position]; + float[] contribution = slots.get(row); + if (contribution != null) { + return contribution; + } + if (remaining.get() <= 0) { + return null; + } + final int hidden = model.hiddenBias.length; + final float[] embedding = model.embeddings[row]; + final int offset = position * model.embeddingSize; + contribution = new float[hidden]; + for (int j = 0; j < hidden; j++) { + final float[] weights = model.hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < model.embeddingSize; d++) { + sum += weights[offset + d] * embedding[d]; + } + contribution[j] = (float) sum; + } + if (slots.compareAndSet(row, null, contribution)) { + remaining.decrementAndGet(); + } else { + contribution = slots.get(row); + } + return contribution; + } + } + /** * Maps the symbolic features of {@link FeedforwardContext} onto embedding rows. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index aa5b64f622..e058d67d1d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -78,6 +78,9 @@ public FeedforwardDependencyParser(FeedforwardDependencyModel model, int beamSiz } this.model = model; this.beamSize = beamSize; + // A parser only ever reads a frozen model, so the scoring cache is safe to turn + // on here; training and refinement work on uncached copies. + model.enableScoringCache(); final String[] outcomes = model.transitions(); this.transitions = new Transition[outcomes.length]; for (int i = 0; i < outcomes.length; i++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index 2d239df018..f431960c09 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -277,6 +277,42 @@ void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws IOException } } + /** + * Pins the scoring cache against the direct path: the parser built in setup turned + * the cache on for the shared model, so scoring the same configuration through a + * fresh uncached copy must agree to float rounding, and the winning transition must + * be identical. Repeated scoring exercises the cache-hit path as well as the + * first-sight fill. + */ + @Test + void testScoringCacheMatchesTheDirectPath() { + final FeedforwardDependencyModel uncached = model.copy(); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = model.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + + for (int round = 0; round < 3; round++) { + final double[] cached = model.score(features); + final double[] direct = uncached.score(features); + assertEquals(direct.length, cached.length); + int bestCached = 0; + int bestDirect = 0; + for (int o = 0; o < cached.length; o++) { + assertEquals(direct[o], cached[o], + Math.max(1.0e-6, Math.abs(direct[o]) * 1.0e-6), + "score " + o + " must agree to float rounding"); + if (cached[o] > cached[bestCached]) { + bestCached = o; + } + if (direct[o] > direct[bestDirect]) { + bestDirect = o; + } + } + assertEquals(bestDirect, bestCached, "the winning transition must be identical"); + } + } + @Test void testNormalizeUsesTheUnicodeDataCaseMapping() { // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's From 9eb6c1b6c164c6e8b680d454ab8bc2c6837f3126 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:15:34 -0400 Subject: [PATCH 75/92] depparse: Validate relation labels through StringUtil.isBlank Replaces the two private blank helpers with the shared predicate; behavior is identical since both already followed the toolkit whitespace definition. --- .../opennlp/tools/depparse/DependencyGraph.java | 17 +---------------- .../java/opennlp/tools/depparse/Transition.java | 17 +---------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index 64921d77f3..5c51d20c24 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -79,7 +79,7 @@ public static DependencyGraph of(int[] heads, String[] relations) { } else if (heads[i] == i) { throw new IllegalArgumentException("token " + i + " must not head itself"); } - if (relations[i] == null || blank(relations[i])) { + if (relations[i] == null || StringUtil.isBlank(relations[i])) { throw new IllegalArgumentException("relation of token " + i + " must not be blank"); } } @@ -89,21 +89,6 @@ public static DependencyGraph of(int[] heads, String[] relations) { return new DependencyGraph(heads.clone(), relations.clone()); } - /** - * Reports whether a relation label is blank under the project whitespace - * definition, which unlike the JDK's includes no-break spaces, so a label spelled - * entirely from them cannot pass as a relation. - */ - private static boolean blank(String value) { - for (int i = 0; i < value.length(); ) { - final int cp = value.codePointAt(i); - if (!StringUtil.isWhitespace(cp)) { - return false; - } - i += Character.charCount(cp); - } - return true; - } /** * @return The number of tokens the graph spans. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java index 2cfa6f9282..d72ea26eb8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -64,26 +64,11 @@ public enum Type { if (label != null) { throw new IllegalArgumentException("a shift must not carry a label: " + label); } - } else if (label == null || blank(label)) { + } else if (label == null || StringUtil.isBlank(label)) { throw new IllegalArgumentException("an arc transition needs a relation label"); } } - /** - * Reports whether a label is blank under the project whitespace definition, which - * unlike the JDK's includes no-break spaces, so a label spelled entirely from them - * cannot pass as a relation. - */ - private static boolean blank(String value) { - for (int i = 0; i < value.length(); ) { - final int cp = value.codePointAt(i); - if (!StringUtil.isWhitespace(cp)) { - return false; - } - i += Character.charCount(cp); - } - return true; - } /** * Creates a left-arc transition. From d894cfc66d036db3751e0bd343078f4c55010afb Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:44:47 -0400 Subject: [PATCH 76/92] dependency: Add a dependency parser manual chapter with a mirror-tested example Add docbkx/dependency.xml, wire it into the manual, and cite ConlluDependencyParserUsageTest. --- opennlp-docs/src/docbkx/dependency.xml | 57 ++++++++++++++++++++++++++ opennlp-docs/src/docbkx/opennlp.xml | 1 + 2 files changed, 58 insertions(+) create mode 100644 opennlp-docs/src/docbkx/dependency.xml diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml new file mode 100644 index 0000000000..a09c1afa54 --- /dev/null +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -0,0 +1,57 @@ + + + + + + + Dependency Parsing + +
+ Introduction + + Dependency parsing assigns each token a syntactic head and a relation label. + DependencyParserME trains on DependencySample streams + and returns a DependencyGraph of heads and relations. CoNLL-U + treebanks are read through ConlluDependencySampleStream. + ConlluDependencyParserUsageTest asserts the train-parse-evaluate + workflow shown here. + +
+ +
+ Training and parsing + + Open a CoNLL-U sample stream, train a model, and parse tokens with their + part-of-speech tags: + + samples = + new ConlluDependencySampleStream(conlluInput, ConlluTagset.U); + +TrainingParameters parameters = TrainingParameters.defaultParams(); +parameters.put(Parameters.CUTOFF_PARAM, 0); +DependencyModel model = DependencyParserME.train("eng", samples, parameters); +DependencyParserME parser = new DependencyParserME(model); + +DependencyGraph graph = parser.parse( + new String[] {"the", "dog", "barks"}, + new String[] {"DET", "NOUN", "VERB"}); +// graph.headOf(0) == 1, graph.relationOf(0) == "det" +// graph.root() is the index of the root token]]> + + A trained model serializes like any other tool model. Evaluation accumulates + unlabeled and labeled attachment scores through + DependencyEvaluator. + +
+ diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 0761fc95ff..4493b16d7d 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -113,6 +113,7 @@ under the License. + From e15e35e475011741d69201c58364c21909533b36 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:05:44 -0400 Subject: [PATCH 77/92] depparse: Address review: decode outcomes once, fold duplication, name constants - DependencyParserME decodes the model outcome inventory once in the constructor and keeps it as a Transition[]. Decoding a sentence now indexes that array instead of parsing an outcome string per configuration and per outcome, and a model trained for another task is rejected with IllegalArgumentException when the parser is built rather than surfacing as an IllegalStateException in the middle of a sentence. Both constructors document the new failure, and a pinning test builds a parser over a MaxentModel whose outcomes are POS tags to prove the rejection happens up front. - Removed the private isBlank copy from ConlluDependencySampleStream and routed sentence separation through StringUtil.isBlank, so the toolkit whitespace definition lives in one place. The rationale the copy carried moved into the nextSentence javadoc, which also gained its missing @return and @throws. DependencyArc validates its relation through the same predicate, so an arc label made only of a no-break space is rejected exactly as a CoNLL-U separator line is. - StringUtil.isBlank rejects a null argument with IllegalArgumentException instead of letting a NullPointerException escape the loop, and documents it. Its test became a parameterized case list plus an explicit null case. - Extracted the magic values of DependencyContextGenerator into named constants: the word/tag and position separators, the feature count the list is sized to, the valency and distance bounds, and the long-distance feature value. The distance feature is computed once rather than twice. - Named the *ROOT* vocabulary key ROOT_SYMBOL and the "*" special-symbol prefix SPECIAL_SYMBOL_PREFIX on FeedforwardDependencyModel and used them from FeedforwardContext and FeedforwardDependencyTrainer, which spelled all three as literals. FeedforwardContext names the first dependent position instead of indexing the template at a bare 6. - Folded the three repeated special-symbol loops in the trainer vocabulary builder into addSpecialSymbols, and hoisted the repeated model.transitions() call out of the transition-decoding loop. - FeedforwardDependencyModel.score and featureIds validate their array argument, and lookup fails loudly when a vocabulary carries no *UNK* row to fall back on instead of returning null and unboxing to a NullPointerException later. - Documented the package-private accessors of FeedforwardDependencyModel, saying which of them hand out the live arrays the trainer writes into. - Trimmed commentary to what the code does: enableScoringCache drops the literature reference and the restatement of how caching pays off, and DependencyEvaluator.processSample uses {@inheritDoc} plus only what the override adds over the Evaluator contract. - Replaced fully qualified java.util.Arrays, java.util.function.Function, java.io.ByteArrayInputStream, java.io.ByteArrayOutputStream and MaxentModel uses with imports in the trainer, the arc-standard state and the tests, and dropped stray blank lines in DependencyGraph and Transition. - Moved the sample() and corpus() helpers, copied verbatim in three test classes, into a shared DependencyTestSamples fixture with the repetition count as a named constant. - Added pinning tests: a relation of U+00A0 alone is rejected while a label such as nmod:poss is kept, an empty DependencySample is rejected, and the arc and graph relation accessors return what was passed in. --- .../opennlp/tools/depparse/DependencyArc.java | 4 +- .../tools/depparse/DependencyGraph.java | 1 - .../tools/depparse/DependencyGraphTest.java | 9 ++ .../tools/depparse/DependencySampleTest.java | 6 ++ .../conllu/ConlluDependencySampleStream.java | 37 +++----- .../tools/depparse/ArcStandardState.java | 5 +- .../depparse/DependencyContextGenerator.java | 74 ++++++++++------ .../tools/depparse/DependencyEvaluator.java | 6 +- .../tools/depparse/DependencyParserME.java | 42 ++++++--- .../tools/depparse/FeedforwardContext.java | 13 ++- .../depparse/FeedforwardDependencyModel.java | 70 ++++++++++++--- .../depparse/FeedforwardDependencyParser.java | 2 +- .../FeedforwardDependencyTrainer.java | 70 +++++++++------ .../opennlp/tools/depparse/Transition.java | 1 - .../DependencyParserEdgeCaseTest.java | 37 +------- .../depparse/DependencyParserMETest.java | 87 +++++++++++++------ .../tools/depparse/DependencyTestSamples.java | 69 +++++++++++++++ .../FeedforwardDependencyParserTest.java | 23 +---- 18 files changed, 362 insertions(+), 194 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java index bf9dc2eea4..1ab520f5fb 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java @@ -17,6 +17,8 @@ package opennlp.tools.depparse; +import opennlp.tools.util.StringUtil; + /** * One labeled edge of a {@link DependencyGraph}: the token at {@code dependent} is governed * by the token at {@code head} under the given {@code relation}. @@ -56,7 +58,7 @@ public record DependencyArc(int head, int dependent, String relation) { if (head == dependent) { throw new IllegalArgumentException("arc must not be a self-loop: " + head); } - if (relation == null || relation.isBlank()) { + if (relation == null || StringUtil.isBlank(relation)) { throw new IllegalArgumentException("relation must not be null or blank"); } } diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index 5c51d20c24..a694c18f3d 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -89,7 +89,6 @@ public static DependencyGraph of(int[] heads, String[] relations) { return new DependencyGraph(heads.clone(), relations.clone()); } - /** * @return The number of tokens the graph spans. */ diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java index 91bf86bd19..91deee1f98 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -120,6 +120,13 @@ void testOutOfRangeHeadThrows() { void testBlankRelationThrows() { assertThrows(IllegalArgumentException.class, () -> DependencyGraph.of(new int[] {1, -1}, new String[] {" ", "root"})); + // blankness follows the toolkit whitespace definition, which covers the no-break + // space U+00A0 that the JDK predicate leaves out + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {"\u00A0", "root"})); + // and a label that only looks unusual is still content + assertEquals("nmod:poss", DependencyGraph.of(new int[] {1, -1}, + new String[] {"nmod:poss", "root"}).relationOf(0)); } @Test @@ -135,6 +142,8 @@ void testArcValidation() { assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, -1, "det")); assertThrows(IllegalArgumentException.class, () -> new DependencyArc(-2, 0, "det")); assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, " ")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, "\u00A0")); assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, null)); + assertEquals("det", new DependencyArc(1, 0, "det").relation()); } } diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java index 03234eab9f..768b1df4e3 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -68,6 +68,12 @@ void testLengthMismatchThrows() { () -> new DependencySample(TOKENS, new String[] {"DT"}, graph())); } + @Test + void testEmptySampleThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[0], new String[0], graph())); + } + @Test void testInputArraysAreCopied() { final String[] tokens = TOKENS.clone(); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 9b5154691b..782bf5fafc 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -108,12 +108,20 @@ public DependencySample read() throws IOException { /** * Reads the syntactic word lines of the next sentence: comments, multiword token * ranges, and empty nodes are dropped; an empty list means the end of the content. + * + *

Sentences are separated by any line {@link StringUtil#isBlank(CharSequence)} + * accepts, so a separator carrying a stray no-break space still separates rather than + * reaching the word line parser.

+ * + * @return The word lines of the next sentence, or an empty list at the end of the + * content. Never {@code null}. + * @throws IOException Thrown if reading fails or a word line has too few columns. */ private List nextSentence() throws IOException { final List words = new ArrayList<>(); String line; while ((line = reader.readLine()) != null) { - if (isBlank(line)) { + if (StringUtil.isBlank(line)) { if (!words.isEmpty()) { return words; } @@ -135,30 +143,11 @@ private List nextSentence() throws IOException { } /** - * Determines whether a line separates two sentences, that is whether it is empty or - * consists entirely of whitespace. - * - *

Blankness is decided with {@link StringUtil#isWhitespace(int)} rather than - * {@link String#isBlank()}, because OpenNLP counts the Unicode {@code Zs} category as - * whitespace while the JDK predicate does not: a separator line carrying a stray - * no-break space is still a separator, not a malformed word line.

+ * Converts one sentence into a sample. * - * @param line The line to inspect. Must not be {@code null}. - * @return {@code true} if the line is blank, {@code false} otherwise. - */ - private static boolean isBlank(String line) { - for (int i = 0; i < line.length(); ) { - final int codePoint = line.codePointAt(i); - if (!StringUtil.isWhitespace(codePoint)) { - return false; - } - i += Character.charCount(codePoint); - } - return true; - } - - /** - * Converts one sentence, or returns {@code null} when its annotation is unusable. + * @param words The word lines of the sentence. + * @return The converted sample, or {@code null} when the sentence's annotation is + * unusable, for example an underscore head or a graph that is not a tree. */ private DependencySample convert(List words) { final int n = words.size(); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index 8f79eeff1f..c3f3e7147e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -17,6 +17,7 @@ package opennlp.tools.depparse; +import java.util.Arrays; /** * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining @@ -72,8 +73,8 @@ public ArcStandardState(int tokenCount) { this.assignedDependents = new int[tokenCount]; this.leftmostDependents = new int[tokenCount]; this.rightmostDependents = new int[tokenCount]; - java.util.Arrays.fill(this.leftmostDependents, NONE); - java.util.Arrays.fill(this.rightmostDependents, NONE); + Arrays.fill(this.leftmostDependents, NONE); + Arrays.fill(this.rightmostDependents, NONE); } private ArcStandardState(ArcStandardState source) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java index 4639bfcbb5..714c1840db 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -35,6 +35,24 @@ public class DependencyContextGenerator { private static final String ROOT_VALUE = "*ROOT*"; private static final String NONE_VALUE = "*NULL*"; + /** Separates a word from the tag of the same position within one feature. */ + private static final char WORD_TAG_SEPARATOR = '/'; + + /** Separates the parts of a feature combining several positions. */ + private static final char POSITION_SEPARATOR = '|'; + + /** The number of features {@link #getContext(ArcStandardState, String[], String[])} emits. */ + private static final int FEATURE_COUNT = 37; + + /** Valency counts at or above this bound share one feature value. */ + private static final int MAX_VALENCY = 3; + + /** Distances at or above this bound share the {@link #LONG_DISTANCE} feature value. */ + private static final int MAX_DISTANCE = 4; + + /** The feature value standing for every distance of {@link #MAX_DISTANCE} or more. */ + private static final String LONG_DISTANCE = "4+"; + /** * Generates the features of the current configuration. * @@ -75,7 +93,7 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag final String s0rcl = dependentRelation(state, s0, false); final String s1rcl = dependentRelation(state, s1, false); - final List features = new ArrayList<>(36); + final List features = new ArrayList<>(FEATURE_COUNT); features.add("s0w=" + s0w); features.add("s0t=" + s0t); features.add("s1w=" + s1w); @@ -86,20 +104,20 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag features.add("b1w=" + b1w); features.add("b1t=" + b1t); features.add("b2t=" + b2t); - features.add("s0wt=" + s0w + '/' + s0t); - features.add("s1wt=" + s1w + '/' + s1t); - features.add("b0wt=" + b0w + '/' + b0t); - features.add("s0w,b0w=" + s0w + '|' + b0w); - features.add("s0t,b0t=" + s0t + '|' + b0t); - features.add("s0w,b0t=" + s0w + '|' + b0t); - features.add("s0t,b0w=" + s0t + '|' + b0w); - features.add("s0wt,b0t=" + s0w + '/' + s0t + '|' + b0t); - features.add("s1t,s0t=" + s1t + '|' + s0t); - features.add("s1t,s0w=" + s1t + '|' + s0w); - features.add("s1w,s0t=" + s1w + '|' + s0t); - features.add("s1t,s0t,b0t=" + s1t + '|' + s0t + '|' + b0t); - features.add("s0t,b0t,b1t=" + s0t + '|' + b0t + '|' + b1t); - features.add("s2t,s1t,s0t=" + s2t + '|' + s1t + '|' + s0t); + features.add("s0wt=" + s0w + WORD_TAG_SEPARATOR + s0t); + features.add("s1wt=" + s1w + WORD_TAG_SEPARATOR + s1t); + features.add("b0wt=" + b0w + WORD_TAG_SEPARATOR + b0t); + features.add("s0w,b0w=" + s0w + POSITION_SEPARATOR + b0w); + features.add("s0t,b0t=" + s0t + POSITION_SEPARATOR + b0t); + features.add("s0w,b0t=" + s0w + POSITION_SEPARATOR + b0t); + features.add("s0t,b0w=" + s0t + POSITION_SEPARATOR + b0w); + features.add("s0wt,b0t=" + s0w + WORD_TAG_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s1t,s0t=" + s1t + POSITION_SEPARATOR + s0t); + features.add("s1t,s0w=" + s1t + POSITION_SEPARATOR + s0w); + features.add("s1w,s0t=" + s1w + POSITION_SEPARATOR + s0t); + features.add("s1t,s0t,b0t=" + s1t + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s0t,b0t,b1t=" + s0t + POSITION_SEPARATOR + b0t + POSITION_SEPARATOR + b1t); + features.add("s2t,s1t,s0t=" + s2t + POSITION_SEPARATOR + s1t + POSITION_SEPARATOR + s0t); features.add("s0lct=" + s0lct); features.add("s0rct=" + s0rct); features.add("s1lct=" + s1lct); @@ -107,23 +125,24 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag features.add("s0lcl=" + s0lcl); features.add("s0rcl=" + s0rcl); features.add("s1rcl=" + s1rcl); - features.add("s1t,s1rct,s0t=" + s1t + '|' + s1rct + '|' + s0t); - features.add("s0t,s0lct,b0t=" + s0t + '|' + s0lct + '|' + b0t); + features.add("s1t,s1rct,s0t=" + s1t + POSITION_SEPARATOR + s1rct + POSITION_SEPARATOR + s0t); + features.add("s0t,s0lct,b0t=" + s0t + POSITION_SEPARATOR + s0lct + POSITION_SEPARATOR + b0t); features.add("s0deps=" + dependents(state, s0)); features.add("s1deps=" + dependents(state, s1)); - features.add("dist=" + distance(s0, b0)); - features.add("dist,s0t,b0t=" + distance(s0, b0) + '|' + s0t + '|' + b0t); + final String distance = distance(s0, b0); + features.add("dist=" + distance); + features.add("dist,s0t,b0t=" + distance + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); return features.toArray(new String[0]); } - private static String word(String[] tokens, int index) { + private String word(String[] tokens, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; } return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; } - private static String tag(String[] tags, int index) { + private String tag(String[] tags, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; } @@ -131,7 +150,7 @@ private static String tag(String[] tags, int index) { } /** The tag of a token's leftmost or rightmost dependent attached so far. */ - private static String dependentTag(ArcStandardState state, String[] tags, int index, + private String dependentTag(ArcStandardState state, String[] tags, int index, boolean leftmost) { if (index < 0) { return NONE_VALUE; @@ -142,7 +161,7 @@ private static String dependentTag(ArcStandardState state, String[] tags, int in } /** The relation of a token's leftmost or rightmost dependent attached so far. */ - private static String dependentRelation(ArcStandardState state, int index, + private String dependentRelation(ArcStandardState state, int index, boolean leftmost) { if (index < 0) { return NONE_VALUE; @@ -156,15 +175,16 @@ private static String dependentRelation(ArcStandardState state, int index, return relation == null ? NONE_VALUE : relation; } - private static String dependents(ArcStandardState state, int index) { - return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), 3)); + private String dependents(ArcStandardState state, int index) { + return index < 0 ? NONE_VALUE + : Integer.toString(Math.min(state.assignedDependents(index), MAX_VALENCY)); } - private static String distance(int s0, int b0) { + private String distance(int s0, int b0) { if (s0 < 0 || b0 < 0) { return NONE_VALUE; } final int distance = b0 - s0; - return distance >= 4 ? "4+" : Integer.toString(distance); + return distance >= MAX_DISTANCE ? LONG_DISTANCE : Integer.toString(distance); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java index f29bada722..c71c538c5f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java @@ -48,10 +48,10 @@ public DependencyEvaluator(DependencyParser parser) { } /** - * Parses the sample's sentence and scores the prediction against the gold graph. + * {@inheritDoc} * - * @param reference The gold sample. Must not be {@code null}. - * @return A {@link DependencySample} carrying the predicted graph. Never {@code null}. + *

The returned sample carries the predicted graph over the reference tokens, and + * every token of the reference contributes to both scores.

*/ @Override protected DependencySample processSample(DependencySample reference) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java index bbd2d219ff..c565e61724 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -44,12 +44,14 @@ public class DependencyParserME implements DependencyParser { private final MaxentModel model; private final DependencyContextGenerator contextGenerator; + private final Transition[] transitions; /** * Initializes a {@link DependencyParserME} from a {@link DependencyModel}. * * @param model The model to parse with. Must not be {@code null}. - * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. */ public DependencyParserME(DependencyModel model) { if (model == null) { @@ -57,13 +59,15 @@ public DependencyParserME(DependencyModel model) { } this.model = model.getParserModel(); this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(this.model); } /** * Initializes a {@link DependencyParserME} with a raw transition model. * * @param model The transition classification model. Must not be {@code null}. - * @throws IllegalArgumentException Thrown if {@code model} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. */ public DependencyParserME(MaxentModel model) { if (model == null) { @@ -71,6 +75,29 @@ public DependencyParserME(MaxentModel model) { } this.model = model; this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(model); + } + + /** + * Decodes the outcome inventory once, so that decoding a sentence indexes it instead + * of parsing an outcome string per configuration and outcome. + * + * @param model The transition classification model. + * @return The transitions by outcome index. Never {@code null}. + * @throws IllegalArgumentException Thrown if an outcome does not decode to a + * transition, which means the model is not a dependency parser model. + */ + private static Transition[] decodeOutcomes(MaxentModel model) { + final Transition[] decoded = new Transition[model.getNumOutcomes()]; + for (int i = 0; i < decoded.length; i++) { + final String outcome = model.getOutcome(i); + try { + decoded[i] = Transition.decode(outcome); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("model outcome is not a transition: " + outcome, e); + } + } + return decoded; } @Override @@ -104,15 +131,8 @@ private Transition bestApplicable(ArcStandardState state, String[] tokens, Strin if (probabilities[i] <= bestProbability) { continue; } - final Transition candidate; - try { - candidate = Transition.decode(model.getOutcome(i)); - } catch (IllegalArgumentException e) { - throw new IllegalStateException( - "model outcome is not a transition: " + model.getOutcome(i), e); - } - if (state.canApply(candidate)) { - best = candidate; + if (state.canApply(transitions[i])) { + best = transitions[i]; bestProbability = probabilities[i]; } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java index 8778071c77..22dec7bf72 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -36,6 +36,9 @@ final class FeedforwardContext { /** The number of dependent positions whose arc label is embedded. */ static final int LABEL_POSITIONS = 8; + /** The index of the first dependent position; the stack and buffer items precede it. */ + private static final int FIRST_DEPENDENT_POSITION = 6; + private FeedforwardContext() { // This class only exposes the static feature template and is never instantiated. } @@ -44,6 +47,12 @@ private FeedforwardContext() { * Extracts the symbolic features of a configuration: {@link #POSITIONS} words, then * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent positions * yield {@code null} entries, which the vocabulary maps to its padding symbol. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The symbolic features, in the order described above. Never {@code null}. */ static String[] extract(ArcStandardState state, String[] tokens, String[] tags) { final int s0 = state.stack(0); @@ -62,7 +71,7 @@ static String[] extract(ArcStandardState state, String[] tokens, String[] tags) features[POSITIONS + i] = symbol(tags, positions[i]); } for (int i = 0; i < LABEL_POSITIONS; i++) { - final int position = positions[6 + i]; + final int position = positions[FIRST_DEPENDENT_POSITION + i]; features[2 * POSITIONS + i] = position >= 0 ? state.assignedRelation(position) : null; } @@ -79,7 +88,7 @@ private static int rightmost(ArcStandardState state, int index) { private static String symbol(String[] values, int index) { if (index == ArcStandardState.ROOT) { - return "*ROOT*"; + return FeedforwardDependencyModel.ROOT_SYMBOL; } return index == ArcStandardState.NONE ? null : values[index]; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index ddd906a880..b881858d50 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -66,9 +66,18 @@ public class FeedforwardDependencyModel { /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the capital. */ private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2'; + /** The vocabulary key of every word, tag, or label the model has no embedding row for. */ static final String UNKNOWN = "*UNK*"; + + /** The vocabulary key of a template position that does not exist in a configuration. */ static final String ABSENT = "*NULL*"; + /** The vocabulary key of the artificial root node. */ + static final String ROOT_SYMBOL = "*ROOT*"; + + /** The prefix marking a vocabulary key as one of the special symbols above. */ + static final String SPECIAL_SYMBOL_PREFIX = "*"; + /** The lazy scoring cache; {@code null} until {@link #enableScoringCache()}. */ private volatile ContributionCache cache; @@ -107,8 +116,12 @@ public class FeedforwardDependencyModel { * {@link #featureIds(String[])}. Must not be {@code null}. * @return One unnormalized score per transition, indexed like * {@link #transitions()}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code features} is {@code null}. */ public double[] score(int[] features) { + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } final int hidden = hiddenBias.length; final double[] h = new double[hidden]; for (int j = 0; j < hidden; j++) { @@ -151,19 +164,15 @@ public double[] score(int[] features) { } /** - * Turns on the scoring cache: the hidden-layer contribution of a (template - * position, embedding row) pair is a fixed vector for a frozen model, so it is - * computed once on first sight and afterwards added instead of being re-derived - * from the embedding on every configuration. Tag and label rows, whose inventories - * are small, are fully cached within a document or two; word rows follow their - * frequency, which is the adaptive form of the precomputation described for this - * architecture by Chen and Manning (2014). + * Turns on the scoring cache: the hidden-layer contribution of a (template position, + * embedding row) pair is a fixed vector while the weights do not change, so it is + * computed once on first sight and afterwards added instead of being re-derived from + * the embedding on every configuration. * *

Cached contributions are rounded to floats once, so scores may differ from the - * uncached path in the last bits; transition decisions are unaffected at any - * realistic margin. The cache is bounded, safe for concurrent readers, and only - * valid on a model whose weights no longer change: training and refinement work on - * uncached copies, and {@link #copy()} never carries a cache over.

+ * uncached path in the last bits. The cache is bounded, safe for concurrent readers, + * and only valid on a model whose weights no longer change: training and refinement + * work on uncached copies, and {@link #copy()} never carries a cache over.

*/ void enableScoringCache() { if (cache == null) { @@ -242,8 +251,12 @@ private float[] contribution(FeedforwardDependencyModel model, int position, int * * @param symbols The symbolic features. Must not be {@code null}. * @return The embedding row per feature. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code symbols} is {@code null}. */ public int[] featureIds(String[] symbols) { + if (symbols == null) { + throw new IllegalArgumentException("symbols must not be null"); + } final int[] ids = new int[symbols.length]; for (int i = 0; i < FeedforwardContext.POSITIONS; i++) { ids[i] = lookup(wordIds, normalize(symbols[i])); @@ -285,7 +298,7 @@ static String normalize(String word) { if (word == null) { return null; } - if (word.startsWith("*")) { + if (word.startsWith(SPECIAL_SYMBOL_PREFIX)) { return word; } int i = 0; @@ -322,6 +335,9 @@ private static int lookup(Map ids, String symbol) { if (id == null) { id = ids.get(UNKNOWN); } + if (id == null) { + throw new IllegalStateException("vocabulary has no " + UNKNOWN + " row to fall back on"); + } return id; } @@ -486,38 +502,68 @@ private static float[][] copyOf(float[][] matrix) { return copy; } + /** + * @return The immutable map from a normalized word to its embedding row. Never {@code null}. + */ Map wordIds() { return wordIds; } + /** + * @return The immutable map from a tag to its embedding row. Never {@code null}. + */ Map tagIds() { return tagIds; } + /** + * @return The immutable map from an arc label to its embedding row. Never {@code null}. + */ Map labelIds() { return labelIds; } + /** + * @return The width of one embedding row. + */ int embeddingSize() { return embeddingSize; } + /** + * @return The live embedding matrix, one row per vocabulary entry, not a copy: the + * trainer writes its updates into it. Never {@code null}. + */ float[][] embeddings() { return embeddings; } + /** + * @return The live hidden layer weights, not a copy. Never {@code null}. + */ float[][] hiddenWeights() { return hiddenWeights; } + /** + * @return The live hidden layer bias, not a copy. Never {@code null}. + */ float[] hiddenBias() { return hiddenBias; } + /** + * @return The live output layer weights, one row per transition, not a copy. Never + * {@code null}. + */ float[][] outputWeights() { return outputWeights; } + /** + * @return The live output layer bias, one entry per transition, not a copy. Never + * {@code null}. + */ float[] outputBias() { return outputBias; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java index e058d67d1d..9cd96699cf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -200,7 +200,7 @@ private DependencyGraph beamParse(String[] tokens, String[] tags) { * @param scores The raw output scores. * @return The log-softmax of {@code scores}. Never {@code null}. */ - private static double[] logSoftmax(double[] scores) { + private double[] logSoftmax(double[] scores) { double max = Double.NEGATIVE_INFINITY; for (final double score : scores) { max = Math.max(max, score); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index d1ec3e840b..38aab67d9c 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -19,10 +19,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -137,7 +139,7 @@ public static FeedforwardDependencyModel train(ObjectStream sa * has the wrong dimensionality. */ public static FeedforwardDependencyModel train(ObjectStream samples, - Settings settings, java.util.function.Function pretrained) + Settings settings, Function pretrained) throws IOException { if (samples == null || settings == null) { throw new IllegalArgumentException("samples and settings must not be null"); @@ -207,11 +209,12 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model while ((sample = samples.read()) != null) { corpus.add(sample); } + final String[] outcomes = model.transitions(); final Map transitionIds = new HashMap<>(); - final Transition[] transitions = new Transition[model.transitions().length]; + final Transition[] transitions = new Transition[outcomes.length]; for (int i = 0; i < transitions.length; i++) { - transitionIds.put(model.transitions()[i], i); - transitions[i] = Transition.decode(model.transitions()[i]); + transitionIds.put(outcomes[i], i); + transitions[i] = Transition.decode(outcomes[i]); } final List trainable = new ArrayList<>(); @@ -428,9 +431,9 @@ private double updateFromCandidates(List candidates) { final double logNormalizer = max + Math.log(normalizer); zero(hiddenGradient); - java.util.Arrays.fill(hiddenBiasGradient, 0.0); + Arrays.fill(hiddenBiasGradient, 0.0); zero(outputGradient); - java.util.Arrays.fill(outputBiasGradient, 0.0); + Arrays.fill(outputBiasGradient, 0.0); embeddingGradients.clear(); for (final BeamNode candidate : candidates) { final double weight = Math.exp(candidate.score - logNormalizer) @@ -514,8 +517,8 @@ private void backward(int[] features, int chosen, double weight) { probabilities[o] = Math.exp(probabilities[o] - max); normalizer += probabilities[o]; } - java.util.Arrays.fill(hiddenDelta, 0.0); - java.util.Arrays.fill(inputDelta, 0.0); + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); final float[][] outputWeights = model.outputWeights(); for (int o = 0; o < outputSize; o++) { // dL/dlogit for a path's step under the conditional likelihood: the path weight @@ -552,7 +555,7 @@ private void backward(int[] features, int chosen, double weight) { } /** Turns raw scores into log-probabilities in place. */ - private static void logSoftmaxInPlace(double[] scores) { + private void logSoftmaxInPlace(double[] scores) { double max = Double.NEGATIVE_INFINITY; for (final double score : scores) { max = Math.max(max, score); @@ -570,10 +573,10 @@ private static void logSoftmaxInPlace(double[] scores) { /** Overwrites the random word rows with pretrained vectors where available. */ private static void seed(FeedforwardDependencyModel model, - java.util.function.Function pretrained, Settings settings) { + Function pretrained, Settings settings) { int seeded = 0; for (final Map.Entry entry : model.wordIds().entrySet()) { - if (entry.getKey().startsWith("*")) { + if (entry.getKey().startsWith(FeedforwardDependencyModel.SPECIAL_SYMBOL_PREFIX)) { // The special unknown, padding, and root symbols have no pretrained // counterpart, so they keep their random initialization. continue; @@ -623,28 +626,22 @@ private static FeedforwardDependencyModel initialize(List corp int row = 0; final Map wordIds = new HashMap<>(); - for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, - FeedforwardDependencyModel.ABSENT, "*ROOT*")) { - wordIds.put(special, row++); - } + row = addSpecialSymbols(wordIds, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); for (final Map.Entry entry : wordCounts.entrySet()) { if (entry.getValue() >= settings.wordCutoff() && !wordIds.containsKey(entry.getKey())) { wordIds.put(entry.getKey(), row++); } } final Map tags = new HashMap<>(); - for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, - FeedforwardDependencyModel.ABSENT, "*ROOT*")) { - tags.put(special, row++); - } + row = addSpecialSymbols(tags, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); for (final String tag : tagIds.keySet()) { tags.put(tag, row++); } final Map labels = new HashMap<>(); - for (final String special : List.of(FeedforwardDependencyModel.UNKNOWN, - FeedforwardDependencyModel.ABSENT)) { - labels.put(special, row++); - } + row = addSpecialSymbols(labels, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT); for (final String label : labelIds.keySet()) { labels.put(label, row++); } @@ -671,6 +668,23 @@ private static FeedforwardDependencyModel initialize(List corp new float[settings.hiddenSize()], outputWeights, new float[transitions.length]); } + /** + * Assigns the next embedding rows to the special symbols of one vocabulary. + * + * @param ids The vocabulary to fill. + * @param row The next free embedding row. + * @param symbols The special symbols, in the order they take their rows. + * @return The next free embedding row after the symbols. + */ + private static int addSpecialSymbols(Map ids, int row, String... symbols) { + int next = row; + for (final String symbol : symbols) { + ids.put(symbol, next); + next++; + } + return next; + } + /** Replays the oracle over every projective sample, emitting one example per step. */ private static void collectExamples(List corpus, FeedforwardDependencyModel model, List featureList, List goldList) { @@ -760,9 +774,9 @@ private static void optimize(FeedforwardDependencyModel model, List featu final int batchEnd = Math.min(batchStart + settings.batchSize(), exampleCount); final int batch = batchEnd - batchStart; zero(hiddenGradient); - java.util.Arrays.fill(hiddenBiasGradient, 0.0); + Arrays.fill(hiddenBiasGradient, 0.0); zero(outputGradient); - java.util.Arrays.fill(outputBiasGradient, 0.0); + Arrays.fill(outputBiasGradient, 0.0); embeddingGradients.clear(); for (int b = batchStart; b < batchEnd; b++) { @@ -810,8 +824,8 @@ private static void optimize(FeedforwardDependencyModel model, List featu } loss -= Math.log(Math.max(probabilities[goldTransition], 1e-12)); - java.util.Arrays.fill(hiddenDelta, 0.0); - java.util.Arrays.fill(inputDelta, 0.0); + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); for (int o = 0; o < outputSize; o++) { final double delta = probabilities[o] - (o == goldTransition ? 1.0 : 0.0); outputBiasGradient[o] += delta; @@ -903,7 +917,7 @@ private static float[][] uniform(Random random, int rows, int columns, double sc private static void zero(double[][] matrix) { for (final double[] row : matrix) { - java.util.Arrays.fill(row, 0.0); + Arrays.fill(row, 0.0); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java index d72ea26eb8..8095e48102 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -69,7 +69,6 @@ public enum Type { } } - /** * Creates a left-arc transition. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java index f7015089e9..4d46e04022 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java @@ -32,6 +32,8 @@ import opennlp.tools.util.Parameters; import opennlp.tools.util.TrainingParameters; +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -48,41 +50,6 @@ public class DependencyParserEdgeCaseTest { private static FeedforwardDependencyModel feedforwardModel; private static FeedforwardDependencyParser feedforwardParser; - /** - * Builds one gold sample from its parallel arrays. - * - * @param tokens The sentence tokens. Must not be {@code null}. - * @param tags The part-of-speech tags aligned with {@code tokens}. - * @param heads The zero-based head per token, {@code -1} for the root. - * @param relations The relation label per token. - * @return The assembled sample. Never {@code null}. - */ - private static DependencySample sample(String[] tokens, String[] tags, int[] heads, - String[] relations) { - return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); - } - - /** - * Builds the projective training corpus: three tiny sentences, each repeated so both - * trainers see enough evidence to memorize them. - * - * @return The training samples. Never {@code null}. - */ - private static List corpus() { - final List distinct = List.of( - sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, - new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), - sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, - new int[] {1, -1}, new String[] {"nsubj", "root"}), - sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, - new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); - final List corpus = new ArrayList<>(); - for (int i = 0; i < 40; i++) { - corpus.addAll(distinct); - } - return corpus; - } - /** * Builds a four-token sample whose gold arcs (2,0) and (3,1) cross, so the tree is * non-projective and has no arc-standard derivation. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java index 62550ad239..716506435c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -17,17 +17,19 @@ package opennlp.tools.depparse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import opennlp.tools.ml.model.MaxentModel; import opennlp.tools.util.ObjectStreamUtils; import opennlp.tools.util.Parameters; import opennlp.tools.util.TrainingParameters; +import static opennlp.tools.depparse.DependencyTestSamples.corpus; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -41,26 +43,6 @@ public class DependencyParserMETest { private static DependencyModel model; private static DependencyParserME parser; - private static DependencySample sample(String[] tokens, String[] tags, int[] heads, - String[] relations) { - return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); - } - - private static List corpus() { - final List distinct = List.of( - sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, - new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), - sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, - new int[] {1, -1}, new String[] {"nsubj", "root"}), - sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, - new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); - final List corpus = new ArrayList<>(); - for (int i = 0; i < 40; i++) { - corpus.addAll(distinct); - } - return corpus; - } - @BeforeAll static void trainParser() throws IOException { final TrainingParameters parameters = TrainingParameters.defaultParams(); @@ -113,7 +95,7 @@ void testConstructorRejectsNullModel() { assertThrows(IllegalArgumentException.class, () -> new DependencyParserME((DependencyModel) null)); assertThrows(IllegalArgumentException.class, - () -> new DependencyParserME((opennlp.tools.ml.model.MaxentModel) null)); + () -> new DependencyParserME((MaxentModel) null)); } @Test @@ -131,10 +113,10 @@ void testTrainValidatesArguments() { @Test void testModelRoundTripThroughSerialization() throws IOException { - final java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); model.serialize(out); final DependencyModel reloaded = new DependencyModel( - new java.io.ByteArrayInputStream(out.toByteArray())); + new ByteArrayInputStream(out.toByteArray())); final DependencyGraph parsed = new DependencyParserME(reloaded) .parse(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}); assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, @@ -146,4 +128,59 @@ void testModelRejectsNullParserModel() { assertThrows(IllegalArgumentException.class, () -> new DependencyModel("eng", null, null)); } + + @Test + void testModelWithForeignOutcomesIsRejectedAtConstruction() { + // The outcome inventory is decoded once up front, so a model trained for another + // task is rejected when the parser is built rather than mid-sentence. + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME(new OutcomeOnlyModel("NN", "VB"))); + } + + /** + * A {@link MaxentModel} that only knows its outcome inventory, enough to build a + * parser from; any other use fails. + */ + private record OutcomeOnlyModel(String... outcomes) implements MaxentModel { + + @Override + public String getOutcome(int i) { + return outcomes[i]; + } + + @Override + public int getNumOutcomes() { + return outcomes.length; + } + + @Override + public double[] eval(String[] context) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, double[] probs) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, float[] values) { + throw new UnsupportedOperationException(); + } + + @Override + public String getBestOutcome(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public String getAllOutcomes(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public int getIndex(String outcome) { + throw new UnsupportedOperationException(); + } + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java new file mode 100644 index 0000000000..051668a585 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * The gold samples shared by the dependency parser tests. + */ +final class DependencyTestSamples { + + /** How often the distinct sentences are repeated in {@link #corpus()}. */ + private static final int REPETITIONS = 40; + + private DependencyTestSamples() { + // This class only exposes static sample builders and is never instantiated. + } + + /** + * Builds one gold sample from its parallel arrays. + * + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @param heads The zero-based head per token, {@code -1} for the root. + * @param relations The relation label per token. + * @return The assembled sample. Never {@code null}. + */ + static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + /** + * Builds the projective training corpus: three tiny sentences, each repeated so both + * trainers see enough evidence to memorize them. + * + * @return The training samples. Never {@code null}. + */ + static List corpus() { + final List distinct = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + final List corpus = new ArrayList<>(REPETITIONS * distinct.size()); + for (int i = 0; i < REPETITIONS; i++) { + corpus.addAll(distinct); + } + return corpus; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index f431960c09..d08426ee38 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -21,7 +21,6 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,6 +30,8 @@ import opennlp.tools.util.ObjectStreamUtils; import opennlp.tools.util.StringUtil; +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -50,26 +51,6 @@ public class FeedforwardDependencyParserTest { private static FeedforwardDependencyModel model; private static FeedforwardDependencyParser parser; - private static DependencySample sample(String[] tokens, String[] tags, int[] heads, - String[] relations) { - return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); - } - - private static List corpus() { - final List distinct = List.of( - sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, - new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), - sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, - new int[] {1, -1}, new String[] {"nsubj", "root"}), - sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, - new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); - final List corpus = new ArrayList<>(); - for (int i = 0; i < 40; i++) { - corpus.addAll(distinct); - } - return corpus; - } - @BeforeAll static void trainParser() throws IOException { // dropout off so the tiny network memorizes deterministically From 62a412dc3e52afb047f7eb54f314be782f1f0681 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:53:28 -0400 Subject: [PATCH 78/92] depparse: Address review: cite specs and document private helpers Add reference links for CoNLL-U, the arc-standard system (Nivre 2004), the feedforward architecture and training recipe (Chen and Manning 2014), early update (Collins and Roark 2004), and the Unicode SpecialCasing file. State that thread safety is implementation specific on the DependencyParser interface. Add javadoc to the remaining private helpers in main and test code. Fold the duplicated corpus reading in the feedforward trainer into a readAll helper, and read the corrupt-model test fixture as UTF-8. The DependencyModel serialVersionUID was verified to equal the serialver default. --- .../tools/depparse/DependencyGraph.java | 9 ++++ .../tools/depparse/DependencyParser.java | 2 + .../tools/depparse/DependencyGraphTest.java | 1 + .../tools/depparse/DependencySampleTest.java | 1 + .../conllu/ConlluDependencySampleStream.java | 11 +++- .../ConlluDependencyParserEvalTest.java | 1 + .../ConlluDependencySampleStreamTest.java | 3 ++ .../tools/depparse/ArcStandardState.java | 20 ++++++- .../depparse/DependencyContextGenerator.java | 4 ++ .../tools/depparse/FeedforwardContext.java | 6 ++- .../depparse/FeedforwardDependencyModel.java | 36 ++++++++++++- .../FeedforwardDependencyTrainer.java | 52 +++++++++++++------ .../tools/depparse/ArcStandardOracleTest.java | 6 +++ .../depparse/DependencyParserMETest.java | 6 +++ .../FeedforwardDependencyParserTest.java | 10 +++- 15 files changed, 143 insertions(+), 25 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java index a694c18f3d..40a1f9c90b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -39,6 +39,9 @@ public final class DependencyGraph { private final int[] heads; private final String[] relations; + /** + * Wraps already validated arrays; instances are created through {@link #of}. + */ private DependencyGraph(int[] heads, String[] relations) { this.heads = heads; this.relations = relations; @@ -144,6 +147,12 @@ public List arcs() { return Collections.unmodifiableList(arcs); } + /** + * Rejects a token index outside {@code [0, size())}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ private void checkIndex(int index) { if (index < 0 || index >= heads.length) { throw new IllegalArgumentException("token index out of range: " + index diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java index 0c6e5d1303..bb9b4029e3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java @@ -27,6 +27,8 @@ * whose indices refer back to the input token array, so spans computed for those tokens * remain valid for the parse.

* + *

Thread safety is implementation specific.

+ * * @see DependencyGraph * @since 3.0.0 */ diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java index 91deee1f98..3229e339f8 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -30,6 +30,7 @@ */ public class DependencyGraphTest { + /** The three-token graph shared by the accessor tests. */ private static DependencyGraph sample() { return DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java index 768b1df4e3..6be16f4817 100644 --- a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -31,6 +31,7 @@ public class DependencySampleTest { private static final String[] TOKENS = {"the", "dog", "barks"}; private static final String[] TAGS = {"DT", "NN", "VBZ"}; + /** The graph matching {@link #TOKENS} and {@link #TAGS}. */ private static DependencyGraph graph() { return DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index 782bf5fafc..e62222768b 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -34,8 +34,9 @@ import opennlp.tools.util.StringUtil; /** - * Reads {@link DependencySample samples} directly from CoNLL-U content, mapping the - * {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * Reads {@link DependencySample samples} directly from + * CoNLL-U content, mapping + * the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. * *

The file is parsed raw, deliberately not through {@link ConlluStream}: that stream * merges multiword token ranges with their syntactic words, which suits the token and @@ -185,6 +186,12 @@ public void close() throws IOException { reader.close(); } + /** + * Opens a fresh UTF-8 reader over the content. + * + * @return A reader positioned at the start of the content. Never {@code null}. + * @throws IOException Thrown if opening the content fails. + */ private BufferedReader open() throws IOException { return new BufferedReader( new InputStreamReader(in.createInputStream(), StandardCharsets.UTF_8)); diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java index af4cc1e919..fd0934ee9f 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserEvalTest.java @@ -77,6 +77,7 @@ void testTrainAndScoreOnUniversalDependencies() throws IOException { assertTrue(evaluator.getLas() > 0.5d, "LAS regressed below the floor"); } + /** Opens a sample stream over one CoNLL-U split using the universal tagset. */ private static ConlluDependencySampleStream samples(Path conllu) throws IOException { final InputStreamFactory in = new MarkableFileInputStreamFactory(conllu.toFile()); return new ConlluDependencySampleStream(in, ConlluTagset.U); diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java index 01dbfaafb9..460e531a7f 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -40,6 +40,7 @@ */ public class ConlluDependencySampleStreamTest { + /** Joins the ten CoNLL-U columns of one word line with tabs. */ private static String line(String... fields) { return String.join("\t", fields); } @@ -68,10 +69,12 @@ private static String line(String... fields) { line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), "") + "\n"; + /** An in-memory factory over the shared fixture. */ private static InputStreamFactory factory() { return () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); } + /** A stream over the shared fixture using the universal tagset. */ private static ConlluDependencySampleStream stream() throws IOException { return new ConlluDependencySampleStream(factory(), ConlluTagset.U); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java index c3f3e7147e..afa2646e9a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -21,7 +21,8 @@ /** * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining - * tokens, and the arcs assigned so far. + * tokens, and the arcs assigned so far. The arc-standard transition system is described + * in Nivre (2004). * *

The stack bottom holds the artificial root, exposed as {@link #ROOT}. Positions that * do not exist, such as the second stack element in the initial configuration, are exposed @@ -77,6 +78,9 @@ public ArcStandardState(int tokenCount) { Arrays.fill(this.rightmostDependents, NONE); } + /** + * Deep-copies {@code source}; used only by {@link #copy()}. + */ private ArcStandardState(ArcStandardState source) { this.tokenCount = source.tokenCount; this.stack = source.stack.clone(); @@ -157,6 +161,14 @@ public void apply(Transition transition) { } } + /** + * Records the arc from {@code head} to {@code dependent} and updates the dependent + * bookkeeping of {@code head} when it is a token rather than the artificial root. + * + * @param head The head token index, or {@link #ROOT} for the artificial root. + * @param dependent The zero-based index of the token being attached. + * @param relation The relation label of the arc. + */ private void attach(int head, int dependent, String relation) { heads[dependent] = head; relations[dependent] = relation; @@ -231,6 +243,12 @@ public int assignedDependents(int index) { return assignedDependents[index]; } + /** + * Rejects a token index outside {@code [0, tokenCount)}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ private void checkTokenIndex(int index) { if (index < 0 || index >= tokenCount) { throw new IllegalArgumentException("token index out of range: " + index); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java index 714c1840db..bcd7ab732d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -135,6 +135,7 @@ public String[] getContext(ArcStandardState state, String[] tokens, String[] tag return features.toArray(new String[0]); } + /** The word at a position, or the marker value for the root and absent positions. */ private String word(String[] tokens, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; @@ -142,6 +143,7 @@ private String word(String[] tokens, int index) { return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; } + /** The tag at a position, or the marker value for the root and absent positions. */ private String tag(String[] tags, int index) { if (index == ArcStandardState.ROOT) { return ROOT_VALUE; @@ -175,11 +177,13 @@ private String dependentRelation(ArcStandardState state, int index, return relation == null ? NONE_VALUE : relation; } + /** A token's dependent count so far, capped at {@link #MAX_VALENCY}. */ private String dependents(ArcStandardState state, int index) { return index < 0 ? NONE_VALUE : Integer.toString(Math.min(state.assignedDependents(index), MAX_VALENCY)); } + /** The bucketed distance between stack top and buffer front, capped at {@link #MAX_DISTANCE}. */ private String distance(int s0, int b0) { if (s0 < 0 || b0 < 0) { return NONE_VALUE; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java index 22dec7bf72..29fb8e4c87 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -20,7 +20,8 @@ /** * The feature template of the feedforward parser: a fixed set of configuration * positions whose words, tags, and arc labels are embedded and concatenated into the - * network input. + * network input, following + * Chen and Manning (2014). * *

Positions: the top three stack and buffer items; the leftmost and rightmost * dependents of the top two stack items; and the leftmost dependent of the leftmost @@ -78,14 +79,17 @@ static String[] extract(ArcStandardState state, String[] tokens, String[] tags) return features; } + /** The leftmost dependent of a position, or the absence marker for absent positions. */ private static int leftmost(ArcStandardState state, int index) { return index >= 0 ? state.leftmostDependent(index) : ArcStandardState.NONE; } + /** The rightmost dependent of a position, or the absence marker for absent positions. */ private static int rightmost(ArcStandardState state, int index) { return index >= 0 ? state.rightmostDependent(index) : ArcStandardState.NONE; } + /** The value at a position: the root symbol for the root, {@code null} when absent. */ private static String symbol(String[] values, int index) { if (index == ArcStandardState.ROOT) { return FeedforwardDependencyModel.ROOT_SYMBOL; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java index b881858d50..dca28791b7 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -38,7 +38,9 @@ /** * The weights of the feedforward transition parser: embeddings for words, tags, and arc * labels, one hidden layer with cube activation, and a transition output layer, stored - * in a plain versioned binary format with no serialization framework involved. + * in a plain versioned binary format with no serialization framework involved. The + * architecture follows + * Chen and Manning (2014). * *

This is the pure-Java neural tier: the network is executed with ordinary array * arithmetic, so parsing needs no native runtime, and the same class scores @@ -283,7 +285,8 @@ public String[] transitions() { *

Case is mapped per code point through UnicodeData, the same mapping as * {@link StringUtil#toLowerCase(CharSequence)}, with one contextual rule on top: a * Greek capital sigma preceded by a letter and not followed by one lowercases to - * the final form U+03C2, the Final_Sigma condition of the Unicode SpecialCasing + * the final form U+03C2, the Final_Sigma condition of the Unicode + * SpecialCasing * file restricted to a single token. Natural lowercase Greek text, and with it * every vocabulary key derived from a treebank, spells a word-final sigma that * way, so without the rule an uppercase Greek word would normalize to a spelling @@ -330,6 +333,14 @@ static String normalize(String word) { return lowered.toString(); } + /** + * Resolves a symbol to its embedding row: absences map to {@link #ABSENT}, symbols + * without a row of their own fall back to {@link #UNKNOWN}. + * + * @param ids The vocabulary to resolve against. + * @param symbol The symbol to resolve, or {@code null} for an absent position. + * @return The embedding row of the symbol or of its fallback. + */ private static int lookup(Map ids, String symbol) { Integer id = ids.get(symbol == null ? ABSENT : symbol); if (id == null) { @@ -414,6 +425,9 @@ public static FeedforwardDependencyModel load(Path path) throws IOException { } } + /** + * Writes one vocabulary as its size followed by (symbol, id) pairs. + */ private static void writeVocabulary(DataOutputStream data, Map ids) throws IOException { data.writeInt(ids.size()); @@ -428,6 +442,9 @@ private static void writeVocabulary(DataOutputStream data, Map } } + /** + * Reads one vocabulary written by {@link #writeVocabulary}. + */ private static Map readVocabulary(DataInputStream data) throws IOException { final int size = data.readInt(); @@ -439,6 +456,9 @@ private static Map readVocabulary(DataInputStream data) return ids; } + /** + * Writes a rectangular matrix as its dimensions followed by its values in row order. + */ private static void writeMatrix(DataOutputStream data, float[][] matrix) throws IOException { data.writeInt(matrix.length); @@ -450,6 +470,9 @@ private static void writeMatrix(DataOutputStream data, float[][] matrix) } } + /** + * Reads a matrix written by {@link #writeMatrix}. + */ private static float[][] readMatrix(DataInputStream data) throws IOException { final int rows = data.readInt(); final int columns = data.readInt(); @@ -462,6 +485,9 @@ private static float[][] readMatrix(DataInputStream data) throws IOException { return matrix; } + /** + * Writes a vector as its length followed by its values. + */ private static void writeVector(DataOutputStream data, float[] vector) throws IOException { data.writeInt(vector.length); for (final float value : vector) { @@ -469,6 +495,9 @@ private static void writeVector(DataOutputStream data, float[] vector) throws IO } } + /** + * Reads a vector written by {@link #writeVector}. + */ private static float[] readVector(DataInputStream data) throws IOException { final float[] vector = new float[data.readInt()]; for (int i = 0; i < vector.length; i++) { @@ -494,6 +523,9 @@ embeddingSize, copyOf(embeddings), copyOf(hiddenWeights), hiddenBias.clone(), copyOf(outputWeights), outputBias.clone()); } + /** + * Deep-copies a matrix, row by row. + */ private static float[][] copyOf(float[][] matrix) { final float[][] copy = new float[matrix.length][]; for (int r = 0; r < matrix.length; r++) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java index 38aab67d9c..7f7e0c07ed 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -34,9 +34,10 @@ /** * Trains the {@link FeedforwardDependencyModel} entirely in Java: oracle-derived * transition examples, minibatch AdaGrad over a softmax cross-entropy loss, cube - * activation, and inverted dropout on the hidden layer. No external training framework - * is involved, so the whole neural tier, training and inference, is plain array - * arithmetic inside the JVM. + * activation, and inverted dropout on the hidden layer, the training recipe of + * Chen and Manning (2014). No external + * training framework is involved, so the whole neural tier, training and inference, is + * plain array arithmetic inside the JVM. * *

Words below the frequency cutoff share a learned unknown embedding; absent * template positions share a learned padding embedding. Non-projective samples have no @@ -144,11 +145,7 @@ public static FeedforwardDependencyModel train(ObjectStream sa if (samples == null || settings == null) { throw new IllegalArgumentException("samples and settings must not be null"); } - final List corpus = new ArrayList<>(); - DependencySample sample; - while ((sample = samples.read()) != null) { - corpus.add(sample); - } + final List corpus = readAll(samples); final FeedforwardDependencyModel model = initialize(corpus, settings); if (pretrained != null) { seed(model, pretrained, settings); @@ -166,10 +163,11 @@ public static FeedforwardDependencyModel train(ObjectStream sa /** * Fine-tunes a locally trained model globally: sentences are decoded with a beam, the * gold derivation is tracked through it, and the moment the gold prefix falls out of - * the beam an early update pushes the model toward keeping it. The loss is a - * conditional likelihood over the beam's candidate paths, scored exactly like the - * beamed parser scores them, summed log-probabilities, so training optimizes the - * quantity decoding uses. + * the beam an early update, in the sense of + * Collins and Roark (2004), pushes + * the model toward keeping it. The loss is a conditional likelihood over the beam's + * candidate paths, scored exactly like the beamed parser scores them, summed + * log-probabilities, so training optimizes the quantity decoding uses. * *

The refined weights are a copy: {@code model} itself is never written to, so a * model already being parsed with, possibly by several threads, keeps behaving exactly @@ -204,11 +202,7 @@ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model if (beamSize < 2) { throw new IllegalArgumentException("beamSize must be at least 2: " + beamSize); } - final List corpus = new ArrayList<>(); - DependencySample sample; - while ((sample = samples.read()) != null) { - corpus.add(sample); - } + final List corpus = readAll(samples); final String[] outcomes = model.transitions(); final Map transitionIds = new HashMap<>(); final Transition[] transitions = new Transition[outcomes.length]; @@ -280,6 +274,7 @@ private static final class BeamNode { private final boolean gold; private ArcStandardState state; + /** Extends {@code parent} by one transition; the start node passes {@code null}. */ private BeamNode(BeamNode parent, int[] features, int transition, double score, boolean gold) { this.parent = parent; @@ -318,6 +313,7 @@ private static final class GlobalOptimizer { private final double[] hiddenDelta; private final double[] inputDelta; + /** Sizes the accumulators and scratch buffers for one refinement run over {@code model}. */ private GlobalOptimizer(FeedforwardDependencyModel model, Settings settings) { this.model = model; this.settings = settings; @@ -571,6 +567,23 @@ private void logSoftmaxInPlace(double[] scores) { } } + /** + * Reads a sample stream into memory; both trainers pass over the corpus repeatedly. + * + * @param samples The stream to drain. + * @return All samples in stream order. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + */ + private static List readAll(ObjectStream samples) + throws IOException { + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + return corpus; + } + /** Overwrites the random word rows with pretrained vectors where available. */ private static void seed(FeedforwardDependencyModel model, Function pretrained, Settings settings) { @@ -880,6 +893,7 @@ private static void optimize(FeedforwardDependencyModel model, List featu } } + /** One AdaGrad step on a weight matrix, with the L2 penalty folded into the gradient. */ private static void update(float[][] weights, double[][] gradients, double[][] accumulators, int batch, Settings settings) { for (int r = 0; r < weights.length; r++) { @@ -895,6 +909,7 @@ private static void update(float[][] weights, double[][] gradients, } } + /** One AdaGrad step on a bias vector; biases carry no L2 penalty. */ private static void updateVector(float[] weights, double[] gradients, double[] accumulators, int batch, Settings settings) { for (int i = 0; i < weights.length; i++) { @@ -905,6 +920,7 @@ private static void updateVector(float[] weights, double[] gradients, } } + /** A matrix drawn uniformly from {@code [-scale, scale]}. */ private static float[][] uniform(Random random, int rows, int columns, double scale) { final float[][] matrix = new float[rows][columns]; for (int r = 0; r < rows; r++) { @@ -915,12 +931,14 @@ private static float[][] uniform(Random random, int rows, int columns, double sc return matrix; } + /** Fills a matrix with zeros. */ private static void zero(double[][] matrix) { for (final double[] row : matrix) { Arrays.fill(row, 0.0); } } + /** Fisher-Yates shuffle of the visit order. */ private static void shuffle(int[] order, Random random) { for (int i = order.length - 1; i > 0; i--) { final int j = random.nextInt(i + 1); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java index 6cfa9f57ca..4ded778794 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java @@ -31,6 +31,12 @@ */ public class ArcStandardOracleTest { + /** + * Derives the oracle transitions for {@code gold} and replays them on a fresh state. + * + * @param gold The gold graph to derive from. + * @return The graph the replayed derivation builds. Never {@code null}. + */ private static DependencyGraph replay(DependencyGraph gold) { final List transitions = ArcStandardOracle.transitions(gold); // every token is shifted once and attached once diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java index 716506435c..aa839c5fbf 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -43,6 +43,12 @@ public class DependencyParserMETest { private static DependencyModel model; private static DependencyParserME parser; + /** + * Trains the shared model once for all tests; the zero cutoff keeps every feature of + * the tiny corpus. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ @BeforeAll static void trainParser() throws IOException { final TrainingParameters parameters = TrainingParameters.defaultParams(); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java index d08426ee38..6d7437587e 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -51,9 +52,14 @@ public class FeedforwardDependencyParserTest { private static FeedforwardDependencyModel model; private static FeedforwardDependencyParser parser; + /** + * Trains the shared model once for all tests, with dropout off and a fixed seed so + * the tiny network memorizes the corpus deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ @BeforeAll static void trainParser() throws IOException { - // dropout off so the tiny network memorizes deterministically final FeedforwardDependencyTrainer.Settings settings = new FeedforwardDependencyTrainer.Settings(16, 32, 120, 32, 0.05, 0.0, 0.0, 1, 17L); model = FeedforwardDependencyTrainer.train( @@ -340,7 +346,7 @@ void testModelRoundTripThroughSerialization() throws IOException { @Test void testCorruptModelFailsLoud() { assertThrows(IOException.class, () -> FeedforwardDependencyModel.load( - new ByteArrayInputStream("not a model".getBytes()))); + new ByteArrayInputStream("not a model".getBytes(StandardCharsets.UTF_8)))); } @Test From 84f059e4dc1055790bf4222d7b1e475fb30c78f9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 21 Aug 2026 20:58:27 -0400 Subject: [PATCH 79/92] OPENNLP-547: Parse CoNLL-U fields without regex --- .../conllu/ConlluDependencySampleStream.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java index e62222768b..f1df227783 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -131,7 +131,7 @@ private List nextSentence() throws IOException { if (line.charAt(0) == '#') { continue; } - final String[] fields = line.split("\t", -1); + final String[] fields = splitFields(line); if (fields.length < COLUMNS) { throw new IOException("not a CoNLL-U word line: " + line); } @@ -143,6 +143,25 @@ private List nextSentence() throws IOException { return words; } + /** + * Splits a CoNLL-U word line into its tab-delimited fields, retaining empty fields. + * + * @param line The line to split. + * @return The fields in source order. Never {@code null}. + */ + private String[] splitFields(String line) { + final List fields = new ArrayList<>(); + int fieldStart = 0; + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) == '\t') { + fields.add(line.substring(fieldStart, i)); + fieldStart = i + 1; + } + } + fields.add(line.substring(fieldStart)); + return fields.toArray(String[]::new); + } + /** * Converts one sentence into a sample. * From 651e9caa365256c7451b923e52bfe650b63fbfda Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 14 Jul 2026 21:29:46 -0400 Subject: [PATCH 80/92] depparse: DependencyAnnotator, the container's first graph-shaped layer Wires the dependency parser into the document pipeline: reads the token and tag layers, parses, and provides a dependencies layer with one DependencyArc per token anchored on the dependent's span. Arc head and dependent are indices into the token layer, exercising the container rule that annotations reference each other by layer and index; the test resolves an arc's head through the token layer back to its span in the original text. (cherry picked from commit 902fbb3f570900d057dcd033ea9ee7c084599911) --- .../tools/depparse/DependencyAnnotator.java | 100 ++++++++++++++++++ .../depparse/DependencyAnnotatorTest.java | 93 ++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java new file mode 100644 index 0000000000..806db6daef --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link DependencyParser} to the document pipeline: reads + * {@link Layers#TOKENS} and {@link Layers#POS_TAGS} and provides + * {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's span. + * + *

This is the first graph-shaped layer: an arc's {@link DependencyArc#head()} and + * {@link DependencyArc#dependent()} are indices into the token layer, following the + * container's rule that annotations reference each other by layer and index, never by + * object identity.

+ * + * @since 3.0.0 + */ +public class DependencyAnnotator implements DocumentAnnotator { + + /** + * Dependency arcs; one annotation per token, aligned with {@link Layers#TOKENS} by + * position, anchored on the dependent token's span. + */ + public static final LayerKey DEPENDENCIES = + LayerKey.of("dependencies", DependencyArc.class); + + private final DependencyParser parser; + + /** + * Initializes the adapter. + * + * @param parser The dependency parser to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyAnnotator(DependencyParser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tokens.isEmpty() || tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final String[] words = new String[tokens.size()]; + final String[] posTags = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + posTags[i] = tags.get(i).value(); + } + final DependencyGraph graph = parser.parse(words, posTags); + final List> arcs = new ArrayList<>(graph.size()); + for (final DependencyArc arc : graph.arcs()) { + arcs.add(new Annotation<>(tokens.get(arc.dependent()).span(), arc)); + } + return document.with(DEPENDENCIES, arcs); + } + + @Override + public Set> requires() { + return Set.of(Layers.TOKENS, Layers.POS_TAGS); + } + + @Override + public Set> provides() { + return Set.of(DEPENDENCIES); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java new file mode 100644 index 0000000000..c047d20c0e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyAnnotator} as the container's first graph-shaped layer: arcs + * reference tokens by layer index, and resolving an arc through the token layer lands on + * the right span of the original text. + */ +public class DependencyAnnotatorTest { + + private static final DependencyParser FIXED = (tokens, tags) -> + DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + + private static Document tokenized() { + return Document.of("the dog barks") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(8, 13), "barks"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "DT"), + new Annotation<>(new Span(4, 7), "NN"), + new Annotation<>(new Span(8, 13), "VBZ"))); + } + + @Test + void testArcsResolveThroughTheTokenLayer() { + final Document document = new DependencyAnnotator(FIXED).annotate(tokenized()); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(3, arcs.size()); + + // the arc of "dog" is anchored on the dependent's span + final Annotation dog = arcs.get(1); + assertEquals(new Span(4, 7), dog.span()); + assertEquals("nsubj", dog.value().relation()); + + // cross-layer reference: the head index resolves into the token layer + final List> tokens = document.get(Layers.TOKENS); + final Annotation head = tokens.get(dog.value().head()); + assertEquals("barks", head.value()); + assertEquals("barks", head.span().getCoveredText(document.text()).toString()); + } + + @Test + void testRootArcCarriesRootHead() { + final Document document = new DependencyAnnotator(FIXED).annotate(tokenized()); + final DependencyArc root = + document.get(DependencyAnnotator.DEPENDENCIES).get(2).value(); + assertEquals(DependencyArc.ROOT_HEAD, root.head()); + assertEquals("root", root.relation()); + } + + @Test + void testMissingLayersThrow() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("no layers"))); + } + + @Test + void testNullParserThrows() { + assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(null)); + } +} From 716e7f3d5bfb35938ccd447d9c810a27451bb5f2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:50:45 -0400 Subject: [PATCH 81/92] depparse: Pipeline and contract tests for the dependency annotator, document-coordinate javadoc --- .../tools/depparse/DependencyAnnotator.java | 28 ++ .../DependencyAnnotatorEdgeCaseTest.java | 119 ++++++++ .../DependencyAnnotatorPipelineTest.java | 288 ++++++++++++++++++ .../depparse/DependencyAnnotatorTest.java | 16 +- 4 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index 806db6daef..9f02d365e0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -37,6 +37,13 @@ * container's rule that annotations reference each other by layer and index, never by * object identity.

* + *

The whole token layer is handed to the parser as one sequence, so the result is a + * single tree over all tokens of the document and the arc indices are positions in the + * document-wide token layer. Because every token span already refers to the original + * document text, anchoring an arc on its dependent token's span puts the arc in document + * coordinates without any offset arithmetic, no matter which sentence the token came + * from.

+ * * @since 3.0.0 */ public class DependencyAnnotator implements DocumentAnnotator { @@ -63,6 +70,23 @@ public DependencyAnnotator(DependencyParser parser) { this.parser = parser; } + /** + * Parses the document's token layer and adds the {@link #DEPENDENCIES} layer. + * + *

The token and tag values are read in layer order and passed to the parser as one + * sequence. The resulting arcs are emitted in token order, so the new layer is aligned + * with {@link Layers#TOKENS} by position, and each arc annotation reuses the span of + * its dependent token.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry a + * non-empty {@link Layers#TOKENS} layer plus a {@link Layers#POS_TAGS} + * layer of equal size. + * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the + * token layer is absent or empty, or the tag layer does not have exactly one + * tag per token. + */ @Override public Document annotate(Document document) { if (document == null) { @@ -74,6 +98,7 @@ public Document annotate(Document document) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } + // unwrap the aligned layers into the parallel arrays the parser interface expects final String[] words = new String[tokens.size()]; final String[] posTags = new String[tokens.size()]; for (int i = 0; i < words.length; i++) { @@ -81,6 +106,9 @@ public Document annotate(Document document) { posTags[i] = tags.get(i).value(); } final DependencyGraph graph = parser.parse(words, posTags); + // graph.arcs() is in token order with one arc per token; anchoring each arc on its + // dependent token's span keeps the layer aligned with the token layer and puts the + // arc in document coordinates, since token spans refer to the original text final List> arcs = new ArrayList<>(graph.size()); for (final DependencyArc arc : graph.arcs()) { arcs.add(new Annotation<>(tokens.get(arc.dependent()).span(), arc)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java new file mode 100644 index 0000000000..2e05260a1d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pins down the boundary behavior of {@link DependencyAnnotator}: the exact exception and + * message for empty and misaligned input layers, the immutability rule that a second + * annotation pass is rejected, and the exact {@code requires()} and {@code provides()} + * declarations the pipeline validation relies on. + */ +public class DependencyAnnotatorEdgeCaseTest { + + /** + * A parser stub that returns a fixed two-token graph regardless of its input, so the + * assertions in this class exercise only the annotator's own layer handling. + */ + private static final DependencyParser FIXED = (tokens, tags) -> + DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, + new String[] {"root", "obj"}); + + /** + * Builds a document over the text {@code "ab cd"} carrying aligned two-entry token and + * tag layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * + * @return A document ready for dependency annotation. Never {@code null}. + */ + private static Document twoTokens() { + return Document.of("ab cd") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + } + + @Test + void testEmptyTokenAndTagLayersAreRejected() { + // a document with zero sentences has zero tokens; the annotator refuses to parse it + final Document empty = Document.of("") + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(empty)); + assertEquals("document needs aligned tokens and pos layers", + e.getMessage()); + } + + @Test + void testMisalignedTagLayerIsRejected() { + // two tokens but only one tag: the layers are present yet not aligned by position + final Document misaligned = Document.of("ab cd") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(misaligned)); + assertEquals("document needs aligned tokens and pos layers", + e.getMessage()); + } + + @Test + void testAnnotatingTwiceIsRejected() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + final Document once = annotator.annotate(twoTokens()); + assertEquals(2, once.get(DependencyAnnotator.DEPENDENCIES).size()); + + // documents are immutable and layers are add-once: a second pass must not overwrite + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(once)); + assertEquals("layer is already present: dependencies", e.getMessage()); + } + + @Test + void testRequiresAndProvidesDeclarationsAreExact() { + final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); + assertEquals(Set.of(Layers.TOKENS, Layers.POS_TAGS), annotator.requires()); + assertEquals(Set.of(DependencyAnnotator.DEPENDENCIES), annotator.provides()); + } + + @Test + void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { + // requires() feeds the analyzer's validation: no tokenizer or tagger, no pipeline + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new DependencyAnnotator(FIXED)); + assertThrows(IllegalArgumentException.class, builder::build); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java new file mode 100644 index 0000000000..f93f22f15f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.document.POSTaggerAnnotator; +import opennlp.tools.document.SentenceDetectorAnnotator; +import opennlp.tools.document.TokenizerAnnotator; +import opennlp.tools.postag.POSTagger; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Demonstrates {@link DependencyAnnotator} at the end of a complete {@link DocumentAnalyzer} + * pipeline: raw text goes in, and the dependency layer comes out anchored on the original + * text. The upstream steps are deliberately simple inline implementations of the task + * interfaces, and the parser is a {@link DependencyParserME} trained here on a tiny corpus + * it can memorize, so every expected head, relation, and span is exact and reproducible. + * + *

The central property under test is coordinate anchoring for multi-sentence input: the + * annotator hands the whole token layer to the parser as one sequence, so the arcs it gets + * back carry indices into the document-wide token layer, and each arc's annotation must sit + * on its dependent token's span in document coordinates. For the second sentence this only + * works when the token layer itself was anchored correctly, which the assertions verify by + * reading the covered text of the arcs' spans back out of the original document.

+ */ +public class DependencyAnnotatorPipelineTest { + + /** + * The two-sentence input text; sentence one covers offsets 0..14 and sentence two covers + * offsets 15..29 of the original document. + */ + private static final String TEXT = "the dog barks. she eats fish."; + + /** + * Maps every token of the test corpus to its part-of-speech tag, standing in for a + * trained tagger. + */ + private static final Map LEXICON = Map.of( + "the", "DT", "dog", "NN", "barks", "VBZ", + "she", "PRP", "eats", "VBZ", "fish", "NN"); + + /** + * A sentence detector stub that closes a sentence after every period and skips the one + * following blank, producing sentence spans in document coordinates. + */ + private static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the pipeline only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A tokenizer stub that treats blanks and periods as token boundaries, so word tokens + * come out without trailing punctuation. + */ + private static final Tokenizer WORD_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the pipeline only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = + i == s.length() || s.charAt(i) == ' ' || s.charAt(i) == '.'; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A dictionary tagger over {@link #LEXICON} that fails loud on any token the test corpus + * does not define, so a tokenization mistake cannot silently degrade the parse. + */ + private static final POSTagger LEXICON_TAGGER = new POSTagger() { + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + final String tag = LEXICON.get(sentence[i]); + if (tag == null) { + throw new IllegalArgumentException("token is not in the test lexicon: " + sentence[i]); + } + tags[i] = tag; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the pipeline only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the pipeline only calls tag"); + } + }; + + private static DependencyParserME parser; + + /** + * Builds the training corpus: the token sequence of the two-sentence document with its + * gold tree, plus a one-token sentence, each repeated often enough for the model to + * memorize them. The document-wide sequence has a single root at {@code barks} with the + * second predicate attached as {@code parataxis}, because a dependency graph always + * forms one tree over the token sequence it is built for. + * + * @return The training samples. Never {@code null} or empty. + */ + private static List corpus() { + final List distinct = List.of( + new DependencySample( + new String[] {"the", "dog", "barks", "she", "eats", "fish"}, + new String[] {"DT", "NN", "VBZ", "PRP", "VBZ", "NN"}, + DependencyGraph.of(new int[] {1, 2, -1, 4, 2, 4}, + new String[] {"det", "nsubj", "root", "nsubj", "parataxis", "obj"})), + new DependencySample(new String[] {"barks"}, new String[] {"VBZ"}, + DependencyGraph.of(new int[] {-1}, new String[] {"root"}))); + final List corpus = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + corpus.addAll(distinct); + } + return corpus; + } + + /** + * Trains the shared parser once for all tests. The trainer is deterministic for a fixed + * corpus and fixed parameters, so the assertions below hold on every run. + * + * @throws IOException Thrown if training fails, which fails the test class. + */ + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + parser = new DependencyParserME(DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(corpus()), parameters)); + } + + /** + * Assembles the complete pipeline: sentence splitting, tokenization, tagging, and + * dependency parsing with the trained model. + * + * @return A {@link DocumentAnalyzer} ready to analyze raw text. Never {@code null}. + */ + private static DocumentAnalyzer pipeline() { + return DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(WORD_TOKENIZER)) + .add(new POSTaggerAnnotator(LEXICON_TAGGER)) + .add(new DependencyAnnotator(parser)) + .build(); + } + + @Test + void testTwoSentenceTextYieldsOneExactArcPerTokenInDocumentCoordinates() { + final Document document = pipeline().analyze(TEXT); + + // sanity of the upstream layers the dependency annotator consumed + assertEquals(2, document.get(Layers.SENTENCES).size()); + assertEquals(6, document.get(Layers.TOKENS).size()); + + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(6, arcs.size()); + + // the memorized gold tree, with every span in document coordinates + final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, 2, 4}; + final String[] relations = {"det", "nsubj", "root", "nsubj", "parataxis", "obj"}; + final Span[] spans = {new Span(0, 3), new Span(4, 7), new Span(8, 13), + new Span(15, 18), new Span(19, 23), new Span(24, 28)}; + for (int i = 0; i < arcs.size(); i++) { + final Annotation arc = arcs.get(i); + assertEquals(spans[i], arc.span(), "span of arc " + i); + assertEquals(heads[i], arc.value().head(), "head of arc " + i); + assertEquals(i, arc.value().dependent(), "dependent of arc " + i); + assertEquals(relations[i], arc.value().relation(), "relation of arc " + i); + } + } + + @Test + void testSecondSentenceArcsResolveToTheOriginalText() { + final Document document = pipeline().analyze(TEXT); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + + // "she" is token 3 of the document-wide token layer, not token 0 of its sentence + final Annotation she = arcs.get(3); + assertEquals(new Span(15, 18), she.span()); + assertEquals("she", she.span().getCoveredText(document.text()).toString()); + assertEquals("nsubj", she.value().relation()); + + // its head index is likewise document-wide: 4 points at "eats", never 1 at "dog" + assertEquals(4, she.value().head()); + final Annotation head = document.get(Layers.TOKENS).get(she.value().head()); + assertEquals("eats", head.value()); + assertEquals(new Span(19, 23), head.span()); + assertEquals("eats", head.span().getCoveredText(document.text()).toString()); + } + + @Test + void testSingleTokenSentenceParsesToARootArc() { + final Document document = pipeline().analyze("barks."); + final List> arcs = + document.get(DependencyAnnotator.DEPENDENCIES); + assertEquals(1, arcs.size()); + assertEquals(new Span(0, 5), arcs.get(0).span()); + assertEquals(DependencyArc.ROOT_HEAD, arcs.get(0).value().head()); + assertEquals(0, arcs.get(0).value().dependent()); + assertEquals("root", arcs.get(0).value().relation()); + } + + @Test + void testTextWithZeroSentencesFailsBeforeTheDependencyAnnotatorRuns() { + // empty text yields no sentences and no tokens, so the tagger already fails loud + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> pipeline().analyze("")); + assertEquals("document lacks the required layer tokens", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java index c047d20c0e..e1f6938d3c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -36,9 +36,20 @@ */ public class DependencyAnnotatorTest { + /** + * A parser stub that always returns the gold graph of {@code "the dog barks"}, so the + * assertions in this class depend only on the annotator's own layer handling and not on + * any trained model. + */ private static final DependencyParser FIXED = (tokens, tags) -> DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + /** + * Builds a document over the text {@code "the dog barks"} carrying aligned token and tag + * layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * + * @return A document ready for dependency annotation. Never {@code null}. + */ private static Document tokenized() { return Document.of("the dog barks") .with(Layers.TOKENS, List.of( @@ -58,12 +69,13 @@ void testArcsResolveThroughTheTokenLayer() { document.get(DependencyAnnotator.DEPENDENCIES); assertEquals(3, arcs.size()); - // the arc of "dog" is anchored on the dependent's span + // the arc of "dog" is anchored on the dependent token's span in the original text final Annotation dog = arcs.get(1); assertEquals(new Span(4, 7), dog.span()); assertEquals("nsubj", dog.value().relation()); - // cross-layer reference: the head index resolves into the token layer + // cross-layer reference: the arc stores its head as an index, and looking that index + // up in the token layer lands on the head token and its span in the original text final List> tokens = document.get(Layers.TOKENS); final Annotation head = tokens.get(dog.value().head()); assertEquals("barks", head.value()); From 96fb60697214a5c4989863320ea807ea3edf8249 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 02:47:16 -0400 Subject: [PATCH 82/92] depparse: Parse each sentence separately in the dependency annotator --- .../tools/depparse/DependencyAnnotator.java | 96 +++++++++++++------ .../DependencyAnnotatorEdgeCaseTest.java | 74 +++++++++++++- .../DependencyAnnotatorPipelineTest.java | 40 ++++---- .../depparse/DependencyAnnotatorTest.java | 7 +- 4 files changed, 166 insertions(+), 51 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index 9f02d365e0..dcc5aebd35 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -29,20 +29,22 @@ /** * Adapts a {@link DependencyParser} to the document pipeline: reads - * {@link Layers#TOKENS} and {@link Layers#POS_TAGS} and provides - * {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's span. + * {@link Layers#SENTENCES}, {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and + * provides {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's + * span. * *

This is the first graph-shaped layer: an arc's {@link DependencyArc#head()} and * {@link DependencyArc#dependent()} are indices into the token layer, following the * container's rule that annotations reference each other by layer and index, never by * object identity.

* - *

The whole token layer is handed to the parser as one sequence, so the result is a - * single tree over all tokens of the document and the arc indices are positions in the - * document-wide token layer. Because every token span already refers to the original - * document text, anchoring an arc on its dependent token's span puts the arc in document - * coordinates without any offset arithmetic, no matter which sentence the token came - * from.

+ *

Each sentence is parsed separately, the way the parser is trained, so every + * sentence gets its own tree and its own root arc. The sentence-local indices the + * parser produces are shifted by the sentence's first token position, which keeps every + * arc's head and dependent a position in the document-wide token layer. Because every + * token span already refers to the original document text, anchoring an arc on its + * dependent token's span puts the arc in document coordinates without further offset + * arithmetic.

* * @since 3.0.0 */ @@ -71,21 +73,25 @@ public DependencyAnnotator(DependencyParser parser) { } /** - * Parses the document's token layer and adds the {@link #DEPENDENCIES} layer. + * Parses the document sentence by sentence and adds the {@link #DEPENDENCIES} layer. * - *

The token and tag values are read in layer order and passed to the parser as one - * sequence. The resulting arcs are emitted in token order, so the new layer is aligned - * with {@link Layers#TOKENS} by position, and each arc annotation reuses the span of - * its dependent token.

+ *

For every sentence, the tokens whose spans lie inside the sentence span are + * passed to the parser with their tags as one sequence, and the resulting + * sentence-local arcs are shifted by the sentence's first token position. Arcs are + * emitted in token order, so the new layer is aligned with {@link Layers#TOKENS} by + * position, and each arc annotation reuses the span of its dependent token. A + * sentence containing no tokens contributes no arcs.

* * @param document The document to annotate. Must not be {@code null} and must carry a - * non-empty {@link Layers#TOKENS} layer plus a {@link Layers#POS_TAGS} - * layer of equal size. + * non-empty {@link Layers#SENTENCES} layer, a non-empty + * {@link Layers#TOKENS} layer whose every token lies inside a + * sentence, and a {@link Layers#POS_TAGS} layer of equal size. * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * token layer is absent or empty, or the tag layer does not have exactly one - * tag per token. + * token layer is absent or empty, the tag layer does not have exactly one + * tag per token, the sentence layer is absent or empty, or a token lies + * outside every sentence. */ @Override public Document annotate(Document document) { @@ -98,27 +104,55 @@ public Document annotate(Document document) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } - // unwrap the aligned layers into the parallel arrays the parser interface expects - final String[] words = new String[tokens.size()]; - final String[] posTags = new String[tokens.size()]; - for (int i = 0; i < words.length; i++) { - words[i] = tokens.get(i).value(); - posTags[i] = tags.get(i).value(); + final List> sentences = document.get(Layers.SENTENCES); + if (sentences.isEmpty()) { + throw new IllegalArgumentException( + "document needs a non-empty " + Layers.SENTENCES + " layer"); } - final DependencyGraph graph = parser.parse(words, posTags); - // graph.arcs() is in token order with one arc per token; anchoring each arc on its - // dependent token's span keeps the layer aligned with the token layer and puts the - // arc in document coordinates, since token spans refer to the original text - final List> arcs = new ArrayList<>(graph.size()); - for (final DependencyArc arc : graph.arcs()) { - arcs.add(new Annotation<>(tokens.get(arc.dependent()).span(), arc)); + final List> arcs = new ArrayList<>(tokens.size()); + // Walk the token layer once: both layers are in text order, so each sentence + // consumes the contiguous run of tokens whose spans it encloses. + int next = 0; + for (final Annotation sentence : sentences) { + final int first = next; + while (next < tokens.size() + && tokens.get(next).span().getStart() >= sentence.span().getStart() + && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) { + next++; + } + final int count = next - first; + if (count == 0) { + continue; + } + // unwrap the sentence's slice into the parallel arrays the parser expects + final String[] words = new String[count]; + final String[] posTags = new String[count]; + for (int i = 0; i < count; i++) { + words[i] = tokens.get(first + i).value(); + posTags[i] = tags.get(first + i).value(); + } + final DependencyGraph graph = parser.parse(words, posTags); + // The parser indexes within the sentence; shifting by the sentence's first token + // position turns every head and dependent into a document-wide token index, and + // anchoring each arc on its dependent token's span puts the arc in document + // coordinates, since token spans refer to the original text. + for (final DependencyArc arc : graph.arcs()) { + final int head = arc.head() == DependencyArc.ROOT_HEAD + ? DependencyArc.ROOT_HEAD : arc.head() + first; + arcs.add(new Annotation<>(tokens.get(first + arc.dependent()).span(), + new DependencyArc(head, arc.dependent() + first, arc.relation()))); + } + } + if (next != tokens.size()) { + throw new IllegalArgumentException("token at " + tokens.get(next).span() + + " lies outside every sentence"); } return document.with(DEPENDENCIES, arcs); } @Override public Set> requires() { - return Set.of(Layers.TOKENS, Layers.POS_TAGS); + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); } @Override diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index 2e05260a1d..e1b33d2a3d 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -55,6 +55,7 @@ public class DependencyAnnotatorEdgeCaseTest { */ private static Document twoTokens() { return Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "ab cd"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 2), "ab"), new Annotation<>(new Span(3, 5), "cd"))) @@ -105,10 +106,81 @@ void testAnnotatingTwiceIsRejected() { @Test void testRequiresAndProvidesDeclarationsAreExact() { final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); - assertEquals(Set.of(Layers.TOKENS, Layers.POS_TAGS), annotator.requires()); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + annotator.requires()); assertEquals(Set.of(DependencyAnnotator.DEPENDENCIES), annotator.provides()); } + /** + * Verifies the per-sentence contract: two one-token sentences are parsed as two + * separate calls, each yielding its own root arc, and the dependents come back as + * document-wide token indices. + */ + @Test + void testEachSentenceGetsItsOwnTree() { + final DependencyParser oneTokenRoot = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + final Document document = Document.of("ab. cd.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "ab."), + new Annotation<>(new Span(4, 7), "cd."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(4, 6), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(4, 6), "VB"))); + + final List> arcs = + new DependencyAnnotator(oneTokenRoot).annotate(document) + .get(DependencyAnnotator.DEPENDENCIES); + assertEquals(2, arcs.size()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 0, "root"), + arcs.get(0).value()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), + arcs.get(1).value()); + assertEquals(new Span(0, 2), arcs.get(0).span()); + assertEquals(new Span(4, 6), arcs.get(1).span()); + } + + /** + * Verifies the sentence-layer requirements fail loud: a token-bearing document + * without a sentence layer is rejected, and so is a token lying outside every + * sentence. + */ + @Test + void testSentenceLayerProblemsAreRejected() { + final Document noSentences = Document.of("ab cd") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + final IllegalArgumentException missing = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(noSentences)); + assertEquals("document needs a non-empty sentences layer", + missing.getMessage()); + + final Document strayToken = Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 2), "ab"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"))); + final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(strayToken)); + assertEquals("token at [3..5) lies outside every sentence", stray.getMessage()); + } + @Test void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { // requires() feeds the analyzer's validation: no tokenizer or tagger, no pipeline diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index f93f22f15f..97991c2754 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -52,11 +52,12 @@ * it can memorize, so every expected head, relation, and span is exact and reproducible. * *

The central property under test is coordinate anchoring for multi-sentence input: the - * annotator hands the whole token layer to the parser as one sequence, so the arcs it gets - * back carry indices into the document-wide token layer, and each arc's annotation must sit - * on its dependent token's span in document coordinates. For the second sentence this only - * works when the token layer itself was anchored correctly, which the assertions verify by - * reading the covered text of the arcs' spans back out of the original document.

+ * annotator parses each sentence separately and shifts the sentence-local arcs by the + * sentence's first token position, so every head and dependent is an index into the + * document-wide token layer and each arc's annotation sits on its dependent token's span in + * document coordinates. For the second sentence this only works when the shift is applied + * consistently, which the assertions verify by reading the covered text of the arcs' spans + * back out of the original document.

*/ public class DependencyAnnotatorPipelineTest { @@ -167,21 +168,25 @@ public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { private static DependencyParserME parser; /** - * Builds the training corpus: the token sequence of the two-sentence document with its - * gold tree, plus a one-token sentence, each repeated often enough for the model to - * memorize them. The document-wide sequence has a single root at {@code barks} with the - * second predicate attached as {@code parataxis}, because a dependency graph always - * forms one tree over the token sequence it is built for. + * Builds the training corpus: the two sentences of the example document with their + * gold trees, plus a one-token sentence, each repeated often enough for the model to + * memorize them. Every sample is one sentence, the unit the annotator hands to the + * parser, so each sentence carries its own root. * * @return The training samples. Never {@code null} or empty. */ private static List corpus() { final List distinct = List.of( new DependencySample( - new String[] {"the", "dog", "barks", "she", "eats", "fish"}, - new String[] {"DT", "NN", "VBZ", "PRP", "VBZ", "NN"}, - DependencyGraph.of(new int[] {1, 2, -1, 4, 2, 4}, - new String[] {"det", "nsubj", "root", "nsubj", "parataxis", "obj"})), + new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}, + DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"})), + new DependencySample( + new String[] {"she", "eats", "fish"}, + new String[] {"PRP", "VBZ", "NN"}, + DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"})), new DependencySample(new String[] {"barks"}, new String[] {"VBZ"}, DependencyGraph.of(new int[] {-1}, new String[] {"root"}))); final List corpus = new ArrayList<>(); @@ -232,9 +237,10 @@ void testTwoSentenceTextYieldsOneExactArcPerTokenInDocumentCoordinates() { document.get(DependencyAnnotator.DEPENDENCIES); assertEquals(6, arcs.size()); - // the memorized gold tree, with every span in document coordinates - final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, 2, 4}; - final String[] relations = {"det", "nsubj", "root", "nsubj", "parataxis", "obj"}; + // the memorized gold trees, one root per sentence, every span and head index in + // document coordinates + final int[] heads = {1, 2, DependencyArc.ROOT_HEAD, 4, DependencyArc.ROOT_HEAD, 4}; + final String[] relations = {"det", "nsubj", "root", "nsubj", "root", "obj"}; final Span[] spans = {new Span(0, 3), new Span(4, 7), new Span(8, 13), new Span(15, 18), new Span(19, 23), new Span(24, 28)}; for (int i = 0; i < arcs.size(); i++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java index e1f6938d3c..554cb3a72d 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -45,13 +45,16 @@ public class DependencyAnnotatorTest { DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); /** - * Builds a document over the text {@code "the dog barks"} carrying aligned token and tag - * layers, mirroring what the upstream tokenizer and tagger annotators would produce. + * Builds a document over the text {@code "the dog barks"} carrying a one-sentence layer plus + * aligned token and tag layers, mirroring what the upstream sentence, tokenizer, and + * tagger annotators would produce. * * @return A document ready for dependency annotation. Never {@code null}. */ private static Document tokenized() { return Document.of("the dog barks") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 13), "the dog barks"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 3), "the"), new Annotation<>(new Span(4, 7), "dog"), From 9bb5ff35ae02736a99e13a1197b190cb9f41ebf1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 00:35:16 -0400 Subject: [PATCH 83/92] depparse: Validate the parsed graph size and pin the annotator's walk boundaries The annotator now rejects a parser that returns a graph over a different token count than its sentence, instead of silently misaligning the dependency layer with the token layer, and the javadoc states the text-order requirement the walk has always relied on. New tests pin the empty-sentence index shift, a token straddling two sentence spans, the stuck-scan path behind a gap token, and the graph-size rejection. The staged copy of the document container was refreshed to the current foundation, whose adapters parse per sentence and whose empty-versus-missing layer distinction moves the empty-text failure into this annotator's own validation. --- .../tools/depparse/DependencyAnnotator.java | 22 ++- .../DependencyAnnotatorEdgeCaseTest.java | 130 +++++++++++++++++- .../DependencyAnnotatorPipelineTest.java | 9 +- 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index dcc5aebd35..eb7db91e60 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -82,16 +82,24 @@ public DependencyAnnotator(DependencyParser parser) { * position, and each arc annotation reuses the span of its dependent token. A * sentence containing no tokens contributes no arcs.

* + *

The sentence and token layers must both be in text order: the walk assigns each + * sentence the contiguous run of tokens its span encloses, so a token that appears + * before its sentence in the layer, or a token overlapping a sentence boundary, is + * reported as lying outside every sentence rather than being silently attached to a + * neighboring sentence.

+ * * @param document The document to annotate. Must not be {@code null} and must carry a - * non-empty {@link Layers#SENTENCES} layer, a non-empty - * {@link Layers#TOKENS} layer whose every token lies inside a - * sentence, and a {@link Layers#POS_TAGS} layer of equal size. + * non-empty {@link Layers#SENTENCES} layer, in text order, a + * non-empty {@link Layers#TOKENS} layer, in text order, whose every + * token lies inside a sentence, and a {@link Layers#POS_TAGS} layer + * of equal size. * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the * token layer is absent or empty, the tag layer does not have exactly one - * tag per token, the sentence layer is absent or empty, or a token lies - * outside every sentence. + * tag per token, the sentence layer is absent or empty, a token lies + * outside every sentence under the text-order walk, or the parser returns a + * graph whose size differs from its sentence's token count. */ @Override public Document annotate(Document document) { @@ -132,6 +140,10 @@ public Document annotate(Document document) { posTags[i] = tags.get(first + i).value(); } final DependencyGraph graph = parser.parse(words, posTags); + if (graph.size() != count) { + throw new IllegalArgumentException("parser returned a graph over " + graph.size() + + " tokens for a sentence of " + count); + } // The parser indexes within the sentence; shifting by the sentence's first token // position turns every head and dependent into a document-wide token index, and // anchoring each arc on its dependent token's span puts the arc in document diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index e1b33d2a3d..d0899f70eb 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -47,6 +47,22 @@ public class DependencyAnnotatorEdgeCaseTest { DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, new String[] {"root", "obj"}); + /** + * A parser stub that returns a flat tree of the requested size, for assertions that + * must get past the annotator's graph-size validation with sentences of any length. + */ + private static final DependencyParser SIZE_MATCHING = (tokens, tags) -> { + final int[] heads = new int[tokens.length]; + final String[] relations = new String[tokens.length]; + heads[0] = DependencyArc.ROOT_HEAD; + relations[0] = "root"; + for (int i = 1; i < heads.length; i++) { + heads[i] = 0; + relations[i] = "dep"; + } + return DependencyGraph.of(heads, relations); + }; + /** * Builds a document over the text {@code "ab cd"} carrying aligned two-entry token and * tag layers, mirroring what the upstream tokenizer and tagger annotators would produce. @@ -177,7 +193,7 @@ void testSentenceLayerProblemsAreRejected() { new Annotation<>(new Span(0, 2), "VB"), new Annotation<>(new Span(3, 5), "NN"))); final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, - () -> new DependencyAnnotator(FIXED).annotate(strayToken)); + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(strayToken)); assertEquals("token at [3..5) lies outside every sentence", stray.getMessage()); } @@ -188,4 +204,116 @@ void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { .add(new DependencyAnnotator(FIXED)); assertThrows(IllegalArgumentException.class, builder::build); } + + /** + * Verifies the javadoc-promised behavior for a sentence containing no tokens: it + * contributes no arcs and no parser call, and the token indices of the sentence + * after it still shift by the correct first-token position rather than by a count + * that includes the empty sentence. + */ + @Test + void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { + final DependencyParser oneTokenRoot = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + final Document document = Document.of("ab. ??? cd.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 3), "ab."), + new Annotation<>(new Span(4, 7), "???"), + new Annotation<>(new Span(8, 11), "cd."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(8, 10), "cd"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(8, 10), "VB"))); + + final List> arcs = + new DependencyAnnotator(oneTokenRoot).annotate(document) + .get(DependencyAnnotator.DEPENDENCIES); + assertEquals(2, arcs.size()); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), + arcs.get(1).value()); + assertEquals(new Span(8, 10), arcs.get(1).span()); + } + + /** + * Verifies the text-order walk on a token straddling two sentence spans: the token + * belongs to neither sentence under the enclosure rule, the scan sticks at it, and + * the annotator reports it as lying outside every sentence instead of silently + * attaching it to one of its neighbors. + */ + @Test + void testTokenStraddlingTwoSentencesIsRejected() { + final Document document = Document.of("ab cd ef") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 4), "ab c"), + new Annotation<>(new Span(4, 8), "d ef"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); + assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies the stuck-scan path: a gap token between sentences stops the walk, and + * the token-bearing sentence after the gap does not pull the scan forward past the + * stray token, which is still reported rather than skipped. + */ + @Test + void testGapTokenBeforeATokenBearingSentenceIsStillRejected() { + final Document document = Document.of("ab cd ef.") + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(6, 9), "ef."))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); + assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + } + + /** + * Verifies that a parser returning a wrong-size graph is rejected loudly instead of + * silently misaligning the dependency layer with the token layer: the fixed + * two-token stub meets a three-token sentence and the annotator names both counts. + */ + @Test + void testWrongSizeGraphFailsLoud() { + final Document document = Document.of("ab cd ef") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 8), "ab cd ef"))) + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd"), + new Annotation<>(new Span(6, 8), "ef"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN"), + new Annotation<>(new Span(6, 8), "NN"))); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(document)); + assertEquals("parser returned a graph over 2 tokens for a sentence of 3", + e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 97991c2754..1adcfbb810 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -285,10 +285,13 @@ void testSingleTokenSentenceParsesToARootArc() { } @Test - void testTextWithZeroSentencesFailsBeforeTheDependencyAnnotatorRuns() { - // empty text yields no sentences and no tokens, so the tagger already fails loud + void testTextWithZeroSentencesFailsAtTheDependencyAnnotator() { + // empty text yields present-but-empty sentence and token layers, which the + // upstream annotators pass through under the empty-versus-missing distinction; + // the dependency annotator itself then refuses to parse an empty token layer final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> pipeline().analyze("")); - assertEquals("document lacks the required layer tokens", e.getMessage()); + assertEquals("document needs aligned tokens and pos layers", + e.getMessage()); } } From 6464aaa31baed02bbbdd044d52228f48a2bcf5db Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 12:29:23 -0400 Subject: [PATCH 84/92] depparse: Mint the dependencies layer key in the toolkit namespace --- .../java/opennlp/tools/depparse/DependencyAnnotator.java | 2 +- .../tools/depparse/DependencyAnnotatorEdgeCaseTest.java | 8 ++++---- .../tools/depparse/DependencyAnnotatorPipelineTest.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index eb7db91e60..dc8282c0ab 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -55,7 +55,7 @@ public class DependencyAnnotator implements DocumentAnnotator { * position, anchored on the dependent token's span. */ public static final LayerKey DEPENDENCIES = - LayerKey.of("dependencies", DependencyArc.class); + Layers.key("dependencies", DependencyArc.class); private final DependencyParser parser; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index d0899f70eb..13a217c844 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -88,7 +88,7 @@ void testEmptyTokenAndTagLayersAreRejected() { .with(Layers.POS_TAGS, List.of()); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(empty)); - assertEquals("document needs aligned tokens and pos layers", + assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", e.getMessage()); } @@ -103,7 +103,7 @@ void testMisalignedTagLayerIsRejected() { new Annotation<>(new Span(0, 2), "VB"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(misaligned)); - assertEquals("document needs aligned tokens and pos layers", + assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", e.getMessage()); } @@ -116,7 +116,7 @@ void testAnnotatingTwiceIsRejected() { // documents are immutable and layers are add-once: a second pass must not overwrite final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> annotator.annotate(once)); - assertEquals("layer is already present: dependencies", e.getMessage()); + assertEquals("layer is already present: opennlp:dependencies", e.getMessage()); } @Test @@ -181,7 +181,7 @@ void testSentenceLayerProblemsAreRejected() { new Annotation<>(new Span(3, 5), "NN"))); final IllegalArgumentException missing = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(noSentences)); - assertEquals("document needs a non-empty sentences layer", + assertEquals("document needs a non-empty opennlp:sentences layer", missing.getMessage()); final Document strayToken = Document.of("ab cd") diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 1adcfbb810..745ef4ef85 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -291,7 +291,7 @@ void testTextWithZeroSentencesFailsAtTheDependencyAnnotator() { // the dependency annotator itself then refuses to parse an empty token layer final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> pipeline().analyze("")); - assertEquals("document needs aligned tokens and pos layers", + assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", e.getMessage()); } } From 4b053016387d5f915e744ea5f355e91ea9c7dd83 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:45:31 -0400 Subject: [PATCH 85/92] dependency: Document the dependency annotator with a mirror-tested example Add a DependencyAnnotator section to the dependency chapter citing DependencyAnnotatorPipelineTest. --- opennlp-docs/src/docbkx/dependency.xml | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index a09c1afa54..71ac5ae7e2 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -54,4 +54,31 @@ DependencyGraph graph = parser.parse( DependencyEvaluator. + +
+ DependencyAnnotator + + DependencyAnnotator adapts a DependencyParser to + the document pipeline. It requires sentence, token, and POS-tag layers, + parses each sentence separately, and shifts sentence-local arcs so every + head and dependent is an index into the document-wide token layer. Each + arc annotation sits on its dependent token's span in original text + coordinates. DependencyAnnotatorPipelineTest asserts the + behavior shown here. + > arcs = + document.get(DependencyAnnotator.DEPENDENCIES); +// one arc per token; second-sentence heads are document-wide indexes +// "she" is token 3 with head 4 ("eats"), span [15..18)]]> + + +
From 367038f7c6ee944f888ce631e36295996375d102 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:54:58 -0400 Subject: [PATCH 86/92] dependency: Align annotator programlisting CDATA with sibling listings Open the CDATA on its own line so the rendered code block has no leading blank line, matching the two listings above it and parser.xml. --- opennlp-docs/src/docbkx/dependency.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 71ac5ae7e2..833440cd92 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -65,8 +65,8 @@ DependencyGraph graph = parser.parse( arc annotation sits on its dependent token's span in original text coordinates. DependencyAnnotatorPipelineTest asserts the behavior shown here. - + Date: Tue, 28 Jul 2026 07:05:38 -0400 Subject: [PATCH 87/92] depparse: Address review: absent-versus-empty layers, javadoc and test cleanup - Check each required layer for presence on its own, so a document missing the sentence, token, or tag layer is rejected with a message that names the key that is absent instead of being folded into the alignment complaint. - Accept present-but-empty required layers: a document with no sentences and no tokens now yields a present-but-empty dependencies layer rather than an IllegalArgumentException, which is the empty-versus-absent distinction the rest of the container annotators already follow. - Extract the shared rejection prefix into a MISSING_LAYER constant so all three absence checks emit one message shape. - Restate the annotate() javadoc and its @throws list against the checks that are actually performed, and say explicitly that the required layers may be empty. - Trim commentary that only repeated the javadoc: the class-level narration about being the first graph-shaped layer, the unwrap-the-slice comment, and the document-coordinate tail of the index-shift comment. - Replace the three hand-rolled absent-layer assertions with a parameterized test over one document per required layer, asserting the message names that layer. - Add pinning tests for the null-document message and for the empty document producing a present-but-empty arc layer, and flip the pipeline test on empty text to assert the same pass-through instead of a failure. - Assert the exact message in testMissingLayersThrow rather than only the exception type. - Hoist the one-token parser stub to a ONE_TOKEN_ROOT constant shared by the two tests that had declared it inline, and extract the repeated STRAY_TOKEN and MISALIGNED expected messages into constants. - Drop the two docbook sentences that pointed readers at test class names, state the present-but-empty layer contract in the annotator section, and correct "indexes" to "indices" in the example comment. --- .../tools/depparse/DependencyAnnotator.java | 68 ++++----- .../DependencyAnnotatorEdgeCaseTest.java | 135 +++++++++++------- .../DependencyAnnotatorPipelineTest.java | 16 +-- .../depparse/DependencyAnnotatorTest.java | 3 +- opennlp-docs/src/docbkx/dependency.xml | 8 +- 5 files changed, 135 insertions(+), 95 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java index dc8282c0ab..e7a7d555fa 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyAnnotator.java @@ -33,18 +33,16 @@ * provides {@link #DEPENDENCIES}, one {@link DependencyArc} per token on the token's * span. * - *

This is the first graph-shaped layer: an arc's {@link DependencyArc#head()} and - * {@link DependencyArc#dependent()} are indices into the token layer, following the - * container's rule that annotations reference each other by layer and index, never by - * object identity.

+ *

An arc's {@link DependencyArc#head()} and {@link DependencyArc#dependent()} are + * indices into the token layer, following the container's rule that annotations + * reference each other by layer and index, never by object identity.

* - *

Each sentence is parsed separately, the way the parser is trained, so every - * sentence gets its own tree and its own root arc. The sentence-local indices the - * parser produces are shifted by the sentence's first token position, which keeps every - * arc's head and dependent a position in the document-wide token layer. Because every - * token span already refers to the original document text, anchoring an arc on its - * dependent token's span puts the arc in document coordinates without further offset - * arithmetic.

+ *

Each sentence is parsed separately, the way the parser contract expects its input, + * so every sentence gets its own tree and its own root arc. The sentence-local indices + * the parser returns are shifted by the sentence's first token position, which keeps + * every arc's head and dependent a position in the document-wide token layer. Token + * spans already refer to the original document text, so anchoring an arc on its + * dependent token's span puts the arc in document coordinates.

* * @since 3.0.0 */ @@ -57,6 +55,9 @@ public class DependencyAnnotator implements DocumentAnnotator { public static final LayerKey DEPENDENCIES = Layers.key("dependencies", DependencyArc.class); + /** The message prefix of every absent-required-layer rejection in this adapter. */ + private static final String MISSING_LAYER = "document lacks the required layer "; + private final DependencyParser parser; /** @@ -79,8 +80,10 @@ public DependencyAnnotator(DependencyParser parser) { * passed to the parser with their tags as one sequence, and the resulting * sentence-local arcs are shifted by the sentence's first token position. Arcs are * emitted in token order, so the new layer is aligned with {@link Layers#TOKENS} by - * position, and each arc annotation reuses the span of its dependent token. A - * sentence containing no tokens contributes no arcs.

+ * position, and each arc annotation reuses the span of its dependent token. The + * required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty dependency layer, and a sentence + * containing no tokens contributes no arcs.

* *

The sentence and token layers must both be in text order: the walk assigns each * sentence the contiguous run of tokens its span encloses, so a token that appears @@ -88,35 +91,39 @@ public DependencyAnnotator(DependencyParser parser) { * reported as lying outside every sentence rather than being silently attached to a * neighboring sentence.

* - * @param document The document to annotate. Must not be {@code null} and must carry a - * non-empty {@link Layers#SENTENCES} layer, in text order, a - * non-empty {@link Layers#TOKENS} layer, in text order, whose every - * token lies inside a sentence, and a {@link Layers#POS_TAGS} layer - * of equal size. + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, in + * text order, and a {@link Layers#POS_TAGS} layer with exactly one + * tag per token, with every token lying inside a sentence. * @return A new {@link Document} with the {@link #DEPENDENCIES} layer added. Never * {@code null}. * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, the - * token layer is absent or empty, the tag layer does not have exactly one - * tag per token, the sentence layer is absent or empty, a token lies - * outside every sentence under the text-order walk, or the parser returns a - * graph whose size differs from its sentence's token count. + * sentence layer, the token layer, or the tag layer is absent, the tag layer + * does not have exactly one tag per token, a token lies outside every + * sentence under the text-order walk, or the parser returns a graph whose + * size differs from its sentence's token count. */ @Override public Document annotate(Document document) { if (document == null) { throw new IllegalArgumentException("document must not be null"); } + if (!document.layers().contains(Layers.SENTENCES)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.SENTENCES); + } + if (!document.layers().contains(Layers.TOKENS)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.TOKENS); + } + if (!document.layers().contains(Layers.POS_TAGS)) { + throw new IllegalArgumentException(MISSING_LAYER + Layers.POS_TAGS); + } + final List> sentences = document.get(Layers.SENTENCES); final List> tokens = document.get(Layers.TOKENS); final List> tags = document.get(Layers.POS_TAGS); - if (tokens.isEmpty() || tags.size() != tokens.size()) { + if (tags.size() != tokens.size()) { throw new IllegalArgumentException("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); } - final List> sentences = document.get(Layers.SENTENCES); - if (sentences.isEmpty()) { - throw new IllegalArgumentException( - "document needs a non-empty " + Layers.SENTENCES + " layer"); - } final List> arcs = new ArrayList<>(tokens.size()); // Walk the token layer once: both layers are in text order, so each sentence // consumes the contiguous run of tokens whose spans it encloses. @@ -132,7 +139,6 @@ public Document annotate(Document document) { if (count == 0) { continue; } - // unwrap the sentence's slice into the parallel arrays the parser expects final String[] words = new String[count]; final String[] posTags = new String[count]; for (int i = 0; i < count; i++) { @@ -145,9 +151,7 @@ public Document annotate(Document document) { + " tokens for a sentence of " + count); } // The parser indexes within the sentence; shifting by the sentence's first token - // position turns every head and dependent into a document-wide token index, and - // anchoring each arc on its dependent token's span puts the arc in document - // coordinates, since token spans refer to the original text. + // position turns every head and dependent into a document-wide token index. for (final DependencyArc arc : graph.arcs()) { final int head = arc.head() == DependencyArc.ROOT_HEAD ? DependencyArc.ROOT_HEAD : arc.head() + first; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java index 13a217c844..fea7b8137c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorEdgeCaseTest.java @@ -19,12 +19,17 @@ import java.util.List; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import opennlp.tools.document.Annotation; import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.LayerKey; import opennlp.tools.document.Layers; import opennlp.tools.util.Span; @@ -33,12 +38,19 @@ /** * Pins down the boundary behavior of {@link DependencyAnnotator}: the exact exception and - * message for empty and misaligned input layers, the immutability rule that a second - * annotation pass is rejected, and the exact {@code requires()} and {@code provides()} - * declarations the pipeline validation relies on. + * message for absent, empty, and misaligned input layers, the immutability rule that a + * second annotation pass is rejected, and the exact {@code requires()} and + * {@code provides()} declarations the pipeline validation relies on. */ public class DependencyAnnotatorEdgeCaseTest { + /** The rejection message for a token that no sentence encloses. */ + private static final String STRAY_TOKEN = "token at [3..5) lies outside every sentence"; + + /** The rejection message for a tag layer that does not have one tag per token. */ + private static final String MISALIGNED = + "document needs aligned opennlp:tokens and opennlp:pos layers"; + /** * A parser stub that returns a fixed two-token graph regardless of its input, so the * assertions in this class exercise only the annotator's own layer handling. @@ -47,6 +59,18 @@ public class DependencyAnnotatorEdgeCaseTest { DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD, 0}, new String[] {"root", "obj"}); + /** + * A parser stub that accepts only one-token sentences and returns their single root + * arc, so a sentence slice of any other length fails the test loudly. + */ + private static final DependencyParser ONE_TOKEN_ROOT = (tokens, tags) -> { + if (tokens.length != 1) { + throw new IllegalStateException("expected one-token sentences, got " + tokens.length); + } + return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, + new String[] {"root"}); + }; + /** * A parser stub that returns a flat tree of the requested size, for assertions that * must get past the annotator's graph-size validation with sentences of any length. @@ -80,22 +104,67 @@ private static Document twoTokens() { new Annotation<>(new Span(3, 5), "NN"))); } + /** + * Supplies one document per required layer, each missing exactly that layer, together + * with the key the rejection message must name. + * + * @return The documents and the expected layer key. Never {@code null}. + */ + private static Stream documentsMissingOneLayer() { + final List> sentence = + List.of(new Annotation<>(new Span(0, 5), "ab cd")); + final List> tokens = List.of( + new Annotation<>(new Span(0, 2), "ab"), + new Annotation<>(new Span(3, 5), "cd")); + final List> tags = List.of( + new Annotation<>(new Span(0, 2), "VB"), + new Annotation<>(new Span(3, 5), "NN")); + return Stream.of( + Arguments.of(Document.of("ab cd") + .with(Layers.TOKENS, tokens).with(Layers.POS_TAGS, tags), Layers.SENTENCES), + Arguments.of(Document.of("ab cd") + .with(Layers.SENTENCES, sentence).with(Layers.POS_TAGS, tags), Layers.TOKENS), + Arguments.of(Document.of("ab cd") + .with(Layers.SENTENCES, sentence).with(Layers.TOKENS, tokens), Layers.POS_TAGS)); + } + + @ParameterizedTest + @MethodSource("documentsMissingOneLayer") + void testAbsentRequiredLayerIsNamed(Document document, LayerKey missing) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(document)); + assertEquals("document lacks the required layer " + missing, e.getMessage()); + } + @Test - void testEmptyTokenAndTagLayersAreRejected() { - // a document with zero sentences has zero tokens; the annotator refuses to parse it + void testNullDocumentIsRejected() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new DependencyAnnotator(FIXED).annotate(null)); + assertEquals("document must not be null", e.getMessage()); + } + + /** + * Verifies the empty-versus-absent distinction of the annotator contract: present but + * empty required layers are valid input and yield a present-but-empty arc layer, so a + * pipeline does not fail on a document without content. + */ + @Test + void testEmptyRequiredLayersYieldAnEmptyArcLayer() { final Document empty = Document.of("") + .with(Layers.SENTENCES, List.of()) .with(Layers.TOKENS, List.of()) .with(Layers.POS_TAGS, List.of()); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> new DependencyAnnotator(FIXED).annotate(empty)); - assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", - e.getMessage()); + final Document annotated = new DependencyAnnotator(FIXED).annotate(empty); + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, + DependencyAnnotator.DEPENDENCIES), annotated.layers()); + assertEquals(List.of(), annotated.get(DependencyAnnotator.DEPENDENCIES)); } @Test void testMisalignedTagLayerIsRejected() { // two tokens but only one tag: the layers are present yet not aligned by position final Document misaligned = Document.of("ab cd") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "ab cd"))) .with(Layers.TOKENS, List.of( new Annotation<>(new Span(0, 2), "ab"), new Annotation<>(new Span(3, 5), "cd"))) @@ -103,8 +172,7 @@ void testMisalignedTagLayerIsRejected() { new Annotation<>(new Span(0, 2), "VB"))); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(FIXED).annotate(misaligned)); - assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", - e.getMessage()); + assertEquals(MISALIGNED, e.getMessage()); } @Test @@ -134,14 +202,6 @@ void testRequiresAndProvidesDeclarationsAreExact() { */ @Test void testEachSentenceGetsItsOwnTree() { - final DependencyParser oneTokenRoot = (tokens, tags) -> { - if (tokens.length != 1) { - throw new IllegalStateException("expected one-token sentences, got " - + tokens.length); - } - return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, - new String[] {"root"}); - }; final Document document = Document.of("ab. cd.") .with(Layers.SENTENCES, List.of( new Annotation<>(new Span(0, 3), "ab."), @@ -154,7 +214,7 @@ void testEachSentenceGetsItsOwnTree() { new Annotation<>(new Span(4, 6), "VB"))); final List> arcs = - new DependencyAnnotator(oneTokenRoot).annotate(document) + new DependencyAnnotator(ONE_TOKEN_ROOT).annotate(document) .get(DependencyAnnotator.DEPENDENCIES); assertEquals(2, arcs.size()); assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 0, "root"), @@ -166,24 +226,11 @@ void testEachSentenceGetsItsOwnTree() { } /** - * Verifies the sentence-layer requirements fail loud: a token-bearing document - * without a sentence layer is rejected, and so is a token lying outside every - * sentence. + * Verifies a token that no sentence encloses is reported instead of being parsed + * outside of any sentence. */ @Test - void testSentenceLayerProblemsAreRejected() { - final Document noSentences = Document.of("ab cd") - .with(Layers.TOKENS, List.of( - new Annotation<>(new Span(0, 2), "ab"), - new Annotation<>(new Span(3, 5), "cd"))) - .with(Layers.POS_TAGS, List.of( - new Annotation<>(new Span(0, 2), "VB"), - new Annotation<>(new Span(3, 5), "NN"))); - final IllegalArgumentException missing = assertThrows(IllegalArgumentException.class, - () -> new DependencyAnnotator(FIXED).annotate(noSentences)); - assertEquals("document needs a non-empty opennlp:sentences layer", - missing.getMessage()); - + void testTokenOutsideEverySentenceIsRejected() { final Document strayToken = Document.of("ab cd") .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 2), "ab"))) .with(Layers.TOKENS, List.of( @@ -194,7 +241,7 @@ void testSentenceLayerProblemsAreRejected() { new Annotation<>(new Span(3, 5), "NN"))); final IllegalArgumentException stray = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(SIZE_MATCHING).annotate(strayToken)); - assertEquals("token at [3..5) lies outside every sentence", stray.getMessage()); + assertEquals(STRAY_TOKEN, stray.getMessage()); } @Test @@ -213,14 +260,6 @@ void testPipelineWithoutUpstreamAnnotatorsFailsAtBuildTime() { */ @Test void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { - final DependencyParser oneTokenRoot = (tokens, tags) -> { - if (tokens.length != 1) { - throw new IllegalStateException("expected one-token sentences, got " - + tokens.length); - } - return DependencyGraph.of(new int[] {DependencyArc.ROOT_HEAD}, - new String[] {"root"}); - }; final Document document = Document.of("ab. ??? cd.") .with(Layers.SENTENCES, List.of( new Annotation<>(new Span(0, 3), "ab."), @@ -234,7 +273,7 @@ void testEmptySentenceContributesNoArcsAndKeepsTheIndexShift() { new Annotation<>(new Span(8, 10), "VB"))); final List> arcs = - new DependencyAnnotator(oneTokenRoot).annotate(document) + new DependencyAnnotator(ONE_TOKEN_ROOT).annotate(document) .get(DependencyAnnotator.DEPENDENCIES); assertEquals(2, arcs.size()); assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 1, "root"), @@ -265,7 +304,7 @@ void testTokenStraddlingTwoSentencesIsRejected() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); - assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + assertEquals(STRAY_TOKEN, e.getMessage()); } /** @@ -290,7 +329,7 @@ void testGapTokenBeforeATokenBearingSentenceIsStillRejected() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new DependencyAnnotator(SIZE_MATCHING).annotate(document)); - assertEquals("token at [3..5) lies outside every sentence", e.getMessage()); + assertEquals(STRAY_TOKEN, e.getMessage()); } /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 745ef4ef85..2147c223f4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -42,7 +42,6 @@ import opennlp.tools.util.TrainingParameters; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; /** * Demonstrates {@link DependencyAnnotator} at the end of a complete {@link DocumentAnalyzer} @@ -285,13 +284,12 @@ void testSingleTokenSentenceParsesToARootArc() { } @Test - void testTextWithZeroSentencesFailsAtTheDependencyAnnotator() { - // empty text yields present-but-empty sentence and token layers, which the - // upstream annotators pass through under the empty-versus-missing distinction; - // the dependency annotator itself then refuses to parse an empty token layer - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> pipeline().analyze("")); - assertEquals("document needs aligned opennlp:tokens and opennlp:pos layers", - e.getMessage()); + void testTextWithZeroSentencesYieldsAnEmptyDependencyLayer() { + // empty text yields present-but-empty sentence, token, and tag layers, which every + // annotator of the pipeline passes through under the empty-versus-absent + // distinction, so the dependency layer comes out present and empty + final Document document = pipeline().analyze(""); + assertEquals(List.of(), document.get(Layers.TOKENS)); + assertEquals(List.of(), document.get(DependencyAnnotator.DEPENDENCIES)); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java index 554cb3a72d..c5b215452b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorTest.java @@ -97,8 +97,9 @@ void testRootArcCarriesRootHead() { @Test void testMissingLayersThrow() { final DependencyAnnotator annotator = new DependencyAnnotator(FIXED); - assertThrows(IllegalArgumentException.class, + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> annotator.annotate(Document.of("no layers"))); + assertEquals("document lacks the required layer " + Layers.SENTENCES, e.getMessage()); } @Test diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 833440cd92..46d01ce2ac 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -24,8 +24,6 @@ DependencyParserME trains on DependencySample streams and returns a DependencyGraph of heads and relations. CoNLL-U treebanks are read through ConlluDependencySampleStream. - ConlluDependencyParserUsageTest asserts the train-parse-evaluate - workflow shown here. @@ -63,8 +61,8 @@ DependencyGraph graph = parser.parse( parses each sentence separately, and shifts sentence-local arcs so every head and dependent is an index into the document-wide token layer. Each arc annotation sits on its dependent token's span in original text - coordinates. DependencyAnnotatorPipelineTest asserts the - behavior shown here. + coordinates. The required layers must be present, but they may be empty: + a document without content yields a present-but-empty arc layer. > arcs = document.get(DependencyAnnotator.DEPENDENCIES); -// one arc per token; second-sentence heads are document-wide indexes +// one arc per token; second-sentence heads are document-wide indices // "she" is token 3 with head 4 ("eats"), span [15..18)]]> From 48137b9ef335a645827554441a30c46577d2482e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:56:08 -0400 Subject: [PATCH 88/92] dependency: Cite the annotator pipeline test in the manual section --- opennlp-docs/src/docbkx/dependency.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 46d01ce2ac..a4e8d31f38 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -63,6 +63,7 @@ DependencyGraph graph = parser.parse( arc annotation sits on its dependent token's span in original text coordinates. The required layers must be present, but they may be empty: a document without content yields a present-but-empty arc layer. + DependencyAnnotatorPipelineTest asserts the behavior shown here. Date: Wed, 26 Aug 2026 07:05:21 -0400 Subject: [PATCH 89/92] OPENNLP-1919: Update annotator imports --- .../tools/depparse/DependencyAnnotatorPipelineTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java index 2147c223f4..46d97fafd4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyAnnotatorPipelineTest.java @@ -29,12 +29,12 @@ import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnalyzer; import opennlp.tools.document.Layers; -import opennlp.tools.document.POSTaggerAnnotator; -import opennlp.tools.document.SentenceDetectorAnnotator; -import opennlp.tools.document.TokenizerAnnotator; import opennlp.tools.postag.POSTagger; +import opennlp.tools.postag.POSTaggerAnnotator; import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.sentdetect.SentenceDetectorAnnotator; import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.TokenizerAnnotator; import opennlp.tools.util.ObjectStreamUtils; import opennlp.tools.util.Parameters; import opennlp.tools.util.Sequence; From 65c215b692b10361b3549af20d7579c5e231aac8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 30 Aug 2026 11:52:44 -0400 Subject: [PATCH 90/92] OPENNLP-1888: Add a ChunkerAnnotator providing a chunks layer Adapts a Chunker to the document pipeline: reads sentences, tokens, and POS tags and provides opennlp:chunks, one annotation per phrase chunk carrying its type on the span of its tokens. --- .../tools/chunker/ChunkerAnnotator.java | 145 ++++++++++++ .../tools/chunker/ChunkerAnnotatorTest.java | 211 ++++++++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java new file mode 100644 index 0000000000..8bd4a17a44 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.chunker; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link Chunker} to the document pipeline: reads {@link Layers#SENTENCES}, + * {@link Layers#TOKENS}, and {@link Layers#POS_TAGS} and provides {@link #CHUNKS}, one + * annotation per phrase chunk carrying the chunk type, for example {@code NP} or + * {@code VP}, on the span from its first to its last token. + * + *

Each sentence is chunked separately with its tokens and tags as one sequence, the + * way the chunker contract expects its input. A chunker's spans index tokens within the + * sentence; the adapter maps them onto the token spans, which already refer to the + * original text, so a chunk covers exactly the text of its tokens. Chunks are emitted in + * text order.

+ * + *

The adapter holds no per-call state; it is as thread-safe as the chunker it + * wraps.

+ * + * @since 3.0.0 + */ +public final class ChunkerAnnotator implements DocumentAnnotator { + + /** + * Phrase chunks; each annotation covers one chunk and carries its type, ordered by + * text position. + */ + public static final LayerKey CHUNKS = Layers.key("chunks", String.class); + + private final Chunker chunker; + + /** + * Initializes the adapter. + * + * @param chunker The chunker to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code chunker} is {@code null}. + */ + public ChunkerAnnotator(Chunker chunker) { + if (chunker == null) { + throw new IllegalArgumentException("chunker must not be null"); + } + this.chunker = chunker; + } + + /** + * Chunks the document sentence by sentence and adds the {@link #CHUNKS} layer. + * + *

The required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty chunk layer. The token and tag + * layers must be aligned one to one.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES}, {@link Layers#TOKENS}, and + * {@link Layers#POS_TAGS} layers, with every token lying inside a + * sentence. + * @return A new {@link Document} with the {@link #CHUNKS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, a + * required layer is absent, the token and tag layers differ in size, a token + * lies outside every sentence, or the chunker returns a span outside the + * sentence, an empty span, or a span without a type. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS, + Layers.POS_TAGS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> tags = document.get(Layers.POS_TAGS); + if (tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final List> chunks = new ArrayList<>(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final String[] sentenceTags = new String[words.length]; + for (int i = 0; i < words.length; i++) { + sentenceTags[i] = tags.get(first + i).value(); + } + for (final Span chunk : chunker.chunkAsSpans(words, sentenceTags)) { + if (chunk.getStart() < 0 || chunk.getEnd() > words.length + || chunk.getStart() >= chunk.getEnd()) { + throw new IllegalArgumentException("chunker returned chunk " + chunk + + " outside the sentence's " + words.length + " tokens"); + } + if (chunk.getType() == null) { + throw new IllegalArgumentException( + "chunker returned chunk " + chunk + " without a type"); + } + chunks.add(new Annotation<>(new Span( + tokens.get(first + chunk.getStart()).span().getStart(), + tokens.get(first + chunk.getEnd() - 1).span().getEnd()), chunk.getType())); + } + }); + return document.with(CHUNKS, chunks); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(CHUNKS); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java new file mode 100644 index 0000000000..3214c13897 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.chunker; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +public class ChunkerAnnotatorTest { + + /** + * A chunker that records every token and tag sequence it receives and answers with + * one {@code NP} chunk per run of {@code N}-initial tags, so slicing and span mapping + * are observable. Tests override {@link #chunkAsSpans(String[], String[])} where a + * deviant answer is the fixture. + */ + private static class RecordingChunker implements Chunker { + + private final List> tokenCalls = new ArrayList<>(); + private final List> tagCalls = new ArrayList<>(); + + @Override + public String[] chunk(String[] toks, String[] tags) { + throw new UnsupportedOperationException("the adapter only calls chunkAsSpans"); + } + + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + tokenCalls.add(List.of(toks)); + tagCalls.add(List.of(tags)); + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= tags.length; i++) { + final boolean noun = i < tags.length && tags[i].startsWith("N"); + if (noun && start < 0) { + start = i; + } else if (!noun && start >= 0) { + spans.add(new Span(start, i, "NP")); + start = -1; + } + } + return spans.toArray(new Span[0]); + } + + @Override + public Sequence[] topKSequences(String[] sentence, String[] tags) { + throw new UnsupportedOperationException("the adapter only calls chunkAsSpans"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, String[] tags, + double minSequenceScore) { + throw new UnsupportedOperationException("the adapter only calls chunkAsSpans"); + } + } + + private static List> tokens(String text, String... forms) { + final List> annotations = new ArrayList<>(forms.length); + int cursor = 0; + for (final String form : forms) { + final int start = text.indexOf(form, cursor); + annotations.add(new Annotation<>(new Span(start, start + form.length()), form)); + cursor = start + form.length(); + } + return annotations; + } + + private static List> values(List> tokens, + String... tags) { + final List> annotations = new ArrayList<>(tags.length); + for (int i = 0; i < tags.length; i++) { + annotations.add(new Annotation<>(tokens.get(i).span(), tags[i])); + } + return annotations; + } + + /** Two sentences whose noun runs straddle neither sentence boundary. */ + private static Document twoSentences() { + final String text = "Mary Jones leads Acme. She joined Acme Corp."; + final List> toks = tokens(text, + "Mary", "Jones", "leads", "Acme", ".", "She", "joined", "Acme", "Corp", "."); + return Document.of(text) + .with(Layers.SENTENCES, List.of( + new Annotation<>(new Span(0, 22), "s"), + new Annotation<>(new Span(23, 44), "s"))) + .with(Layers.TOKENS, toks) + .with(Layers.POS_TAGS, values(toks, + "NNP", "NNP", "VBZ", "NNP", ".", "PRP", "VBD", "NNP", "NNP", ".")); + } + + @Test + void testChunksEachSentenceOntoTokenSpans() { + final RecordingChunker chunker = new RecordingChunker(); + final Document document = new ChunkerAnnotator(chunker).annotate(twoSentences()); + + Assertions.assertEquals(List.of( + List.of("Mary", "Jones", "leads", "Acme", "."), + List.of("She", "joined", "Acme", "Corp", ".")), chunker.tokenCalls); + Assertions.assertEquals(List.of( + List.of("NNP", "NNP", "VBZ", "NNP", "."), + List.of("PRP", "VBD", "NNP", "NNP", ".")), chunker.tagCalls); + final List> chunks = document.get(ChunkerAnnotator.CHUNKS); + Assertions.assertEquals(List.of( + new Annotation<>(new Span(0, 10), "NP"), + new Annotation<>(new Span(17, 21), "NP"), + new Annotation<>(new Span(34, 43), "NP")), chunks); + Assertions.assertEquals("Acme Corp", document.text().subSequence(34, 43).toString()); + } + + @Test + void testEmptyLayersYieldEmptyChunkLayer() { + final Document document = new ChunkerAnnotator(new RecordingChunker()).annotate( + Document.of("") + .with(Layers.SENTENCES, List.of()) + .with(Layers.TOKENS, List.of()) + .with(Layers.POS_TAGS, List.of())); + Assertions.assertTrue(document.layers().contains(ChunkerAnnotator.CHUNKS)); + Assertions.assertTrue(document.get(ChunkerAnnotator.CHUNKS).isEmpty()); + } + + @Test + void testLayerContract() { + final ChunkerAnnotator annotator = new ChunkerAnnotator(new RecordingChunker()); + Assertions.assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS), + annotator.requires()); + Assertions.assertEquals(Set.of(ChunkerAnnotator.CHUNKS), annotator.provides()); + Assertions.assertEquals("opennlp:chunks", ChunkerAnnotator.CHUNKS.id()); + Assertions.assertEquals("ChunkerAnnotator", annotator.toString()); + } + + @Test + void testRejectsNullChunkerAndMissingLayers() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new ChunkerAnnotator(null)); + final ChunkerAnnotator annotator = new ChunkerAnnotator(new RecordingChunker()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(null)); + final Document untagged = Document.of("Mary.") + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "s"))) + .with(Layers.TOKENS, tokens("Mary.", "Mary", ".")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(untagged)); + } + + @Test + void testRejectsMisalignedTagLayer() { + final String text = "Mary."; + final List> toks = tokens(text, "Mary", "."); + final Document document = Document.of(text) + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "s"))) + .with(Layers.TOKENS, toks) + .with(Layers.POS_TAGS, values(toks, "NNP")); + final ChunkerAnnotator annotator = new ChunkerAnnotator(new RecordingChunker()); + final IllegalArgumentException rejection = Assertions.assertThrows( + IllegalArgumentException.class, () -> annotator.annotate(document)); + Assertions.assertTrue(rejection.getMessage().contains("aligned"), + rejection.getMessage()); + } + + @Test + void testRejectsChunksOutsideSentenceOrWithoutType() { + final ChunkerAnnotator outside = new ChunkerAnnotator(new RecordingChunker() { + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + return new Span[] {new Span(0, toks.length + 1, "NP")}; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> outside.annotate(twoSentences())); + final ChunkerAnnotator empty = new ChunkerAnnotator(new RecordingChunker() { + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + return new Span[] {new Span(1, 1, "NP")}; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> empty.annotate(twoSentences())); + final ChunkerAnnotator untyped = new ChunkerAnnotator(new RecordingChunker() { + @Override + public Span[] chunkAsSpans(String[] toks, String[] tags) { + return new Span[] {new Span(0, 1)}; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> untyped.annotate(twoSentences())); + } +} From 1b24daddd615f81be030148b750a76422552254a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 30 Aug 2026 11:52:44 -0400 Subject: [PATCH 91/92] OPENNLP-1888: Add a ParserAnnotator providing a phrases layer Adapts a constituency Parser to the document pipeline: reads sentences and tokens and provides opennlp:phrases, one annotation per phrase node above the part-of-speech level in pre-order, carrying the label and the span of the head token the parser's head rules select. --- .../opennlp/tools/parser/ParserAnnotator.java | 208 ++++++++++++++++++ .../tools/parser/ParserAnnotatorTest.java | 158 +++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java new file mode 100644 index 0000000000..7e81717b6d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.DocumentAnnotators; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Adapts a constituency {@link Parser} to the document pipeline: reads + * {@link Layers#SENTENCES} and {@link Layers#TOKENS} and provides {@link #PHRASES}, one + * annotation per phrase node of each sentence's parse, carrying the phrase label and the + * span of its head token. + * + *

Each sentence is parsed from its tokens as one sequence. Every node above the + * part-of-speech level except the root becomes an annotation on the span from its first + * to its last token, in pre-order, so an enclosing phrase precedes the phrases it + * contains and phrases nest by span containment. Part-of-speech nodes are left to the + * {@link Layers#POS_TAGS} layer and token nodes to {@link Layers#TOKENS}. The head token + * is the one the parser's head rules select, so a consumer can read the head of a noun + * phrase without its own rules.

+ * + *

The adapter holds no per-call state; it is as thread-safe as the parser it + * wraps.

+ * + * @since 3.0.0 + */ +public final class ParserAnnotator implements DocumentAnnotator { + + /** + * One phrase of a constituency parse: its label, such as {@code NP} or {@code VP}, + * and the span of the token that heads it. The phrase's own span is the annotation's + * span. + * + * @param label The phrase label. Must not be {@code null} or blank. + * @param head The span of the head token in the document text. Must not be + * {@code null}. + * + * @since 3.0.0 + */ + public record Phrase(String label, Span head) { + + /** + * Validates the phrase. + * + * @throws IllegalArgumentException Thrown if {@code label} is {@code null} or + * blank, or {@code head} is {@code null}. + */ + public Phrase { + if (label == null || StringUtil.isBlank(label)) { + throw new IllegalArgumentException("label must not be null or blank"); + } + if (head == null) { + throw new IllegalArgumentException("head must not be null"); + } + } + } + + /** + * Parse phrases; each annotation covers one phrase and carries its {@link Phrase}, + * in pre-order of the parse tree. + */ + public static final LayerKey PHRASES = Layers.key("phrases", Phrase.class); + + private final Parser parser; + + /** + * Initializes the adapter. + * + * @param parser The parser to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public ParserAnnotator(Parser parser) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + this.parser = parser; + } + + /** + * Parses the document sentence by sentence and adds the {@link #PHRASES} layer. + * + *

The required layers must be present, but they may be empty: a document without + * sentences or tokens yields a present-but-empty phrase layer.

+ * + * @param document The document to annotate. Must not be {@code null} and must carry + * the {@link Layers#SENTENCES} and {@link Layers#TOKENS} layers, with + * every token lying inside a sentence. + * @return A new {@link Document} with the {@link #PHRASES} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, a + * required layer is absent, a token lies outside every sentence, or the + * parser returns a node outside the sentence's tokens. + */ + @Override + public Document annotate(Document document) { + DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS); + final List> sentences = document.get(Layers.SENTENCES); + final List> tokens = document.get(Layers.TOKENS); + final List> phrases = new ArrayList<>(); + DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> { + final Parse root = parser.parse(Parse.createFromTokens(words)); + if (root == null) { + throw new IllegalArgumentException("parser returned no parse"); + } + // The parse text is the tokens joined by single spaces, so a token's start in + // that text identifies its index. + final int[] starts = new int[words.length]; + for (int i = 1; i < words.length; i++) { + starts[i] = starts[i - 1] + words[i - 1].length() + 1; + } + final int length = starts[words.length - 1] + words[words.length - 1].length(); + for (final Parse child : root.getChildren()) { + collect(child, first, starts, length, tokens, phrases); + } + }); + return document.with(PHRASES, phrases); + } + + /** Emits a node and, in pre-order, every phrase node below it. */ + private static void collect(Parse node, int first, int[] starts, int length, + List> tokens, List> phrases) { + if (node.isPosTag() || Parser.TOK_NODE.equals(node.getType())) { + return; + } + final int from = tokenIndex(starts, length, node.getSpan().getStart(), node); + final int to = tokenIndex(starts, length, node.getSpan().getEnd(), node); + final int head = node.getHeadIndex(); + if (head < 0 || head >= starts.length) { + throw new IllegalArgumentException("parser returned node " + node.getType() + + " with head " + head + " outside the sentence's " + starts.length + " tokens"); + } + phrases.add(new Annotation<>(new Span(tokens.get(first + from).span().getStart(), + tokens.get(first + to).span().getEnd()), + new Phrase(node.getType(), tokens.get(first + head).span()))); + for (final Parse child : node.getChildren()) { + collect(child, first, starts, length, tokens, phrases); + } + } + + /** + * Maps an offset in the parse text, the tokens joined by single spaces, to the index + * of the token it lies in or, for a span end, ends. + */ + private static int tokenIndex(int[] starts, int length, int offset, Parse node) { + if (offset < 0 || offset > length) { + throw new IllegalArgumentException("parser returned node " + node.getType() + + " at " + node.getSpan() + " outside the sentence's " + starts.length + + " tokens"); + } + int low = 0; + int high = starts.length - 1; + while (low < high) { + final int mid = (low + high + 1) >>> 1; + if (starts[mid] <= offset) { + low = mid; + } else { + high = mid - 1; + } + } + return low; + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.SENTENCES, Layers.TOKENS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(PHRASES); + } + + /** + * {@return the adapter's simple class name, which names it in pipeline validation + * messages} + */ + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java new file mode 100644 index 0000000000..dee78a2ef7 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.parser.ParserAnnotator.Phrase; +import opennlp.tools.util.Span; + +public class ParserAnnotatorTest { + + /** + * A parser that builds one fixed bracketing over any sentence of six tokens, + * {@code (S (NP (NP 0 1) (PP 2 (NP 3))) (VP 4) 5)}, with explicit heads, so the + * span and head mapping is observable without a model. + */ + private static class FixedParser implements Parser { + + @Override + public Parse[] parse(Parse tokens, int numParses) { + return new Parse[] {parse(tokens)}; + } + + @Override + public Parse parse(Parse tokens) { + final Parse[] toks = tokens.getChildren(); + if (toks.length != 6) { + return tokens; + } + final String[] tags = {"DT", "NN", "IN", "NNP", "VBD", "."}; + final Parse[] pos = new Parse[toks.length]; + for (int i = 0; i < toks.length; i++) { + pos[i] = node(tokens, tags[i], toks[i], toks[i], toks[i]); + tokens.insert(pos[i]); + } + final Parse innerNp = node(tokens, "NP", pos[0], pos[1], toks[1]); + tokens.insert(innerNp); + final Parse maryNp = node(tokens, "NP", pos[3], pos[3], toks[3]); + tokens.insert(maryNp); + final Parse pp = node(tokens, "PP", pos[2], maryNp, toks[2]); + tokens.insert(pp); + final Parse outerNp = node(tokens, "NP", innerNp, pp, toks[1]); + tokens.insert(outerNp); + final Parse vp = node(tokens, "VP", pos[4], pos[4], toks[4]); + tokens.insert(vp); + final Parse s = node(tokens, "S", outerNp, pos[5], toks[4]); + tokens.insert(s); + return tokens; + } + + private static Parse node(Parse root, String type, Parse from, Parse to, Parse head) { + return new Parse(root.getText(), + new Span(from.getSpan().getStart(), to.getSpan().getEnd()), type, 1.0, head); + } + } + + private static List> tokens(String text, String... forms) { + final List> annotations = new ArrayList<>(forms.length); + int cursor = 0; + for (final String form : forms) { + final int start = text.indexOf(form, cursor); + annotations.add(new Annotation<>(new Span(start, start + form.length()), form)); + cursor = start + form.length(); + } + return annotations; + } + + /** One six-token sentence whose text carries a double space the parse text lacks. */ + private static Document sentence() { + final String text = "The dog of Mary ran."; + return Document.of(text) + .with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 21), "s"))) + .with(Layers.TOKENS, tokens(text, "The", "dog", "of", "Mary", "ran", ".")); + } + + @Test + void testEmitsPhrasesInPreOrderOnTokenSpansWithHeads() { + final Document document = new ParserAnnotator(new FixedParser()).annotate(sentence()); + + final List> phrases = document.get(ParserAnnotator.PHRASES); + Assertions.assertEquals(List.of("S", "NP", "NP", "PP", "NP", "VP"), + phrases.stream().map(a -> a.value().label()).toList()); + Assertions.assertEquals(List.of( + new Span(0, 21), new Span(0, 16), new Span(0, 7), new Span(9, 16), + new Span(12, 16), new Span(17, 20)), + phrases.stream().map(Annotation::span).toList()); + Assertions.assertEquals("The dog of Mary", document.text().subSequence(0, 16).toString()); + final Span dog = new Span(4, 7); + final Span ran = new Span(17, 20); + Assertions.assertEquals(List.of(ran, dog, dog, new Span(9, 11), new Span(12, 16), ran), + phrases.stream().map(a -> a.value().head()).toList()); + } + + @Test + void testEmptyLayersYieldEmptyPhraseLayer() { + final Document document = new ParserAnnotator(new FixedParser()).annotate( + Document.of("").with(Layers.SENTENCES, List.of()).with(Layers.TOKENS, List.of())); + Assertions.assertTrue(document.layers().contains(ParserAnnotator.PHRASES)); + Assertions.assertTrue(document.get(ParserAnnotator.PHRASES).isEmpty()); + } + + @Test + void testLayerContract() { + final ParserAnnotator annotator = new ParserAnnotator(new FixedParser()); + Assertions.assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS), annotator.requires()); + Assertions.assertEquals(Set.of(ParserAnnotator.PHRASES), annotator.provides()); + Assertions.assertEquals("opennlp:phrases", ParserAnnotator.PHRASES.id()); + Assertions.assertEquals("ParserAnnotator", annotator.toString()); + } + + @Test + void testRejectsNullParserMissingLayersAndNullParse() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new ParserAnnotator(null)); + final ParserAnnotator annotator = new ParserAnnotator(new FixedParser()); + Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(Document.of("x"))); + final ParserAnnotator silent = new ParserAnnotator(new FixedParser() { + @Override + public Parse parse(Parse tokens) { + return null; + } + }); + Assertions.assertThrows(IllegalArgumentException.class, + () -> silent.annotate(sentence())); + } + + @Test + void testPhraseRejectsBlankLabelAndNullHead() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Phrase(" ", new Span(0, 1))); + Assertions.assertThrows(IllegalArgumentException.class, () -> new Phrase("NP", null)); + } +} From e37f9c10874da650d61dffbb80564b4d528eecfc Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 30 Aug 2026 11:52:44 -0400 Subject: [PATCH 92/92] OPENNLP-1888: List the chunker and parser adapters in the manual Also makes the ParserAnnotator helpers instance methods, matching the other adapters. --- .../src/main/java/opennlp/tools/parser/ParserAnnotator.java | 4 ++-- opennlp-docs/src/docbkx/document.xml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java index 7e81717b6d..e38b2fdb22 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java @@ -142,7 +142,7 @@ public Document annotate(Document document) { } /** Emits a node and, in pre-order, every phrase node below it. */ - private static void collect(Parse node, int first, int[] starts, int length, + private void collect(Parse node, int first, int[] starts, int length, List> tokens, List> phrases) { if (node.isPosTag() || Parser.TOK_NODE.equals(node.getType())) { return; @@ -166,7 +166,7 @@ private static void collect(Parse node, int first, int[] starts, int length, * Maps an offset in the parse text, the tokens joined by single spaces, to the index * of the token it lies in or, for a span end, ends. */ - private static int tokenIndex(int[] starts, int length, int offset, Parse node) { + private int tokenIndex(int[] starts, int length, int offset, Parse node) { if (offset < 0 || offset > length) { throw new IllegalArgumentException("parser returned node " + node.getType() + " at " + node.getSpan() + " outside the sentence's " + starts.length diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 9a1aa92410..76c626ada5 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -108,6 +108,7 @@ String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]> Adapters for the toolkit's own components are provided: SentenceDetectorAnnotator, TokenizerAnnotator, POSTaggerAnnotator, NameFinderAnnotator, + ChunkerAnnotator, ParserAnnotator, LemmatizerAnnotator, and StemmerAnnotator. Each wraps one of the existing component APIs; an application that needs only one task can keep using that component's API directly. The following pipeline combines