diff --git a/.agents/skills/testing-rdf-roundtrip/SKILL.md b/.agents/skills/testing-rdf-roundtrip/SKILL.md index c62375184e..2c7a031985 100644 --- a/.agents/skills/testing-rdf-roundtrip/SKILL.md +++ b/.agents/skills/testing-rdf-roundtrip/SKILL.md @@ -287,7 +287,7 @@ directory alone matches nothing and `sysml` then complains about a missing exten ## The corpus round-trip ratchet (run it before and after any writer/encoder change) -`TestCorpusRoundTrip` (`tests/corpus/roundtrip_test.go`) runs the three-hop trip +`TestExamplesSweep` (`tests/corpus/examples_sweep_test.go`, verdicts in `roundtrip_test.go`) runs the three-hop trip over **every** `.sysml`/`.kerml` under `examples/` — the 32 committed models, the 100-file training corpus and the three pilot corpora (213 files) — and pins one verdict per file in `tests/corpus/testdata/corpus_roundtrip_expected.txt`. It runs in about two seconds. @@ -296,7 +296,7 @@ Record: `docs/project/rdf-corpus-roundtrip.md`. ```bash ./scripts/download-training-examples.sh && ./scripts/download-pilot-corpora.sh # once OPENSYSML_REQUIRE_TRAINING_CORPUS=1 OPENSYSML_REQUIRE_PILOT_CORPORA=1 \ - go test -count=1 -v ./tests/corpus -run TestCorpusRoundTrip + go test -count=1 -v ./tests/corpus -run TestExamplesSweep ``` Without the require variables an absent corpus **skips** the gate with a `GATE NOT RUN` banner on @@ -321,7 +321,7 @@ regression does, so every movement is adjudicated. When your change moves files: record. 3. Regenerate and commit the expectation file in the same PR: ```bash - go test ./tests/corpus -run TestCorpusRoundTrip -update-corpus-roundtrip + go test ./tests/corpus -run TestExamplesSweep -update-corpus-roundtrip git diff --stat tests/corpus/testdata/corpus_roundtrip_expected.txt ``` Run the update twice and confirm the second run leaves the file unchanged; the run is diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e4dceffa9..f98951cc1d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -705,7 +705,7 @@ jobs: go-race-test: executor: go-executor resource_class: large - parallelism: 19 + parallelism: 17 steps: - checkout - rehearsal-checkout-tag @@ -721,14 +721,14 @@ jobs: - install-fmpy # internal/exec/runtime (31-57 minutes whole, near 8.5 GB under -race), tests/export and tests/corpus - # (about 40 minutes whole) are split by test name as make test-shard does: four and three shards. + # (about 40 minutes whole) are split by test name as make test-shard does: four and two shards. - run: name: Run Go race tests command: | set -o pipefail # One entry per node; a quoted entry runs its shards in turn (tests/hygiene keeps # the entries the shard list of scripts/race-shard.sh). - shards=(runtime runtime-corpus model smt "corpus codegen" corpus-roundtrip corpus-apijson export export-library export-examples export-corpus export-settled graph translate grpc tools rest "migrate runtime-exploration" cli) + shards=(runtime runtime-corpus model smt "corpus codegen" corpus-sweep export export-library export-examples export-corpus graph translate grpc tools rest "migrate runtime-exploration" cli) if [ "$CIRCLE_NODE_TOTAL" != "${#shards[@]}" ]; then echo "error: parallelism is $CIRCLE_NODE_TOTAL, but the shard array has ${#shards[@]} nodes" >&2 exit 1 @@ -743,31 +743,26 @@ jobs: exploration='^TestExploreWithIsExploreOverTheConformanceCorpus$' corpus='^Test(ExploreWithIsExploreOverTheConformanceCorpus|CheckAgreesWithExploreOverTheConformanceCorpus|CheckWitnessesReplayOverTheConformanceCorpus|ExecutionConformance|ExecutionConformanceUnderPolicies)$' exportcorpus='^TestAPIJSONWritersNeverEmitNameReferences$' - exportsettled='^TestSettledCollectionsAreWhatReconcilingWouldFind$' exportlibrary='^TestLibraryFilesComeBackFromTheGraphAlone$' exportexamples='^TestExamplesCorpusInvariants$' - # The Makefile's RACE_CORPUS_ROUNDTRIP and RACE_CORPUS_APIJSON; tests/hygiene keeps them the same. - corpusroundtrip='^TestCorpusRoundTrip$' - corpusapijson='^TestCorpusAPIJSONRoundTrip$' + # The Makefile's RACE_CORPUS_SWEEP; tests/hygiene keeps the two the same. + corpussweep='^TestExamplesSweep$' case "$shard" in # Its own shard and process: alone it nears 5 GB, the five suites together 9 GB under -race, # over this class's 8 GB. The Makefile's RACE_RUNTIME_EXPLORATION; tests/hygiene keeps the two the same. runtime-exploration) go test -json -race -pgo=off -timeout 30m -run "$exploration" $pkgs | stream ;; runtime-corpus) go test -json -race -pgo=off -timeout 30m -run "$corpus" -skip "$exploration" $pkgs | stream ;; runtime) go test -json -race -pgo=off -timeout 30m -skip "$corpus" $pkgs | stream ;; - # The Makefile's RACE_EXPORT_CORPUS, RACE_EXPORT_SETTLED, RACE_EXPORT_LIBRARY and RACE_EXPORT_EXAMPLES; tests/hygiene - # keeps them the same. tests/export whole passed 55 minutes under -race; its remainder ran 37, so five shards. + # The Makefile's RACE_EXPORT_CORPUS, RACE_EXPORT_LIBRARY and RACE_EXPORT_EXAMPLES; tests/hygiene + # keeps them the same. tests/export whole passed 55 minutes under -race; its remainder ran 37, so four shards. export-corpus) go test -json -race -pgo=off -timeout 55m -run "$exportcorpus" $pkgs | stream ;; - # Four settled models at once peak at 8.2-8.4 GB under -race, this class's 8 GB (killed on - # develop at #2014); two at once peak at 6.5 GB for a tenth more wall time. - export-settled) go test -json -race -pgo=off -timeout 55m -parallel 2 -run "$exportsettled" $pkgs | stream ;; export-library) go test -json -race -pgo=off -timeout 55m -run "$exportlibrary" $pkgs | stream ;; export-examples) go test -json -race -pgo=off -timeout 55m -run "$exportexamples" $pkgs | stream ;; - export) go test -json -race -pgo=off -timeout 55m -skip "$exportcorpus|$exportsettled|$exportlibrary|$exportexamples" $pkgs | stream ;; - # tests/corpus's two round trips over every example ran about 20 minutes each under -race. - corpus-roundtrip) go test -json -race -pgo=off -timeout 55m -run "$corpusroundtrip" $pkgs | stream ;; - corpus-apijson) go test -json -race -pgo=off -timeout 55m -run "$corpusapijson" $pkgs | stream ;; - corpus) go test -json -race -pgo=off -timeout 30m -skip "$corpusroundtrip|$corpusapijson" $pkgs | stream ;; + export) go test -json -race -pgo=off -timeout 55m -skip "$exportcorpus|$exportlibrary|$exportexamples" $pkgs | stream ;; + # tests/corpus's sweep has -parallel models in hand at once, a whole corpus directory as one model + # among them: four peaked at 10.4 GB and three at 10.0 GB under -race on a four-vCPU box (this class ran ~0.8x of that). + corpus-sweep) go test -json -race -pgo=off -timeout 55m -parallel 3 -run "$corpussweep" $pkgs | stream ;; + corpus) go test -json -race -pgo=off -timeout 30m -skip "$corpussweep" $pkgs | stream ;; tools) go test -json -race -pgo=off -timeout 30m $pkgs | stream go test -C tools -json -race -pgo=off -timeout 30m ./... | stream @@ -811,7 +806,7 @@ jobs: executor: go-executor resource_class: large # The race test's shards but the runtime's corpus ones: without -race the runtime runs whole in about 19 minutes. - parallelism: 19 + parallelism: 17 steps: - checkout - rehearsal-checkout-tag @@ -834,7 +829,7 @@ jobs: command: | # tests/export and tests/corpus whole overran 30 minutes, so make coverage-shard splits them by test name. # cmd/sysml has its own node: its exploration budgets took 23-30 minutes on the rest node. - shards=(runtime model smt corpus corpus-roundtrip corpus-apijson export export-library export-examples export-corpus export-settled graph codegen translate grpc tools rest migrate cli) + shards=(runtime model smt corpus corpus-sweep export export-library export-examples export-corpus graph codegen translate grpc tools rest migrate cli) if [ "$CIRCLE_NODE_TOTAL" != "${#shards[@]}" ]; then echo "error: parallelism is $CIRCLE_NODE_TOTAL, but the Makefile's COVERAGE_SHARDS has ${#shards[@]} shards" >&2 exit 1 @@ -1022,32 +1017,17 @@ jobs: environment: OPENSYSML_REQUIRE_PILOT_LIBRARY_XMI: "1" - # The RDF round-trip ratchet over every example alone, so its per-verdict - # counts are legible in the log and a skip cannot hide behind a green suite. + # The RDF and api-json round-trip ratchets over every example alone, so their + # per-verdict counts are legible in the log and a skip cannot hide behind a green suite. - run: - name: Run RDF corpus round-trip gate + name: Run corpus round-trip gates command: | set -o pipefail - go test -count=1 -v ./tests/corpus -run 'TestCorpusRoundTrip' | tee corpus-roundtrip-gate.log - grep -E 'corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log + go test -count=1 -v ./tests/corpus -run 'TestExamplesSweep' | tee corpus-roundtrip-gate.log + grep -E ': corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log + grep -E 'api-json corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log if grep -qE '^\s*--- SKIP' corpus-roundtrip-gate.log; then - echo "error: the RDF corpus round-trip gate skipped" >&2 - exit 1 - fi - environment: - OPENSYSML_REQUIRE_TRAINING_CORPUS: "1" - OPENSYSML_REQUIRE_PILOT_CORPORA: "1" - OPENSYSML_REQUIRE_SPEC_EXAMPLES: "1" - - # The same ratchet over the API's JSON element form, beside the RDF one. - - run: - name: Run api-json corpus round-trip gate - command: | - set -o pipefail - go test -count=1 -v ./tests/corpus -run 'TestCorpusAPIJSONRoundTrip' | tee api-json-roundtrip-gate.log - grep -E 'api-json corpus round trip: [0-9]+ files:' api-json-roundtrip-gate.log - if grep -qE '^\s*--- SKIP' api-json-roundtrip-gate.log; then - echo "error: the api-json corpus round-trip gate skipped" >&2 + echo "error: the corpus round-trip gates skipped" >&2 exit 1 fi environment: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 28014d95ce..610c408a16 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -107,9 +107,9 @@ jobs: TIER: ${{ steps.areas.outputs.tier }} run: | if [ "$TIER" != quick ]; then - echo 'go_shards=["runtime","runtime-corpus","runtime-exploration","model","smt","corpus","corpus-roundtrip","corpus-apijson","export","export-library","export-examples","export-corpus","export-settled","graph","codegen","translate","grpc","rest","migrate","cli"]' + echo 'go_shards=["runtime","runtime-corpus","runtime-exploration","model","smt","corpus","corpus-sweep","export","export-library","export-examples","export-corpus","graph","codegen","translate","grpc","rest","migrate","cli"]' else - echo 'go_shards=["runtime runtime-corpus runtime-exploration model","codegen migrate","cli translate","graph","smt","corpus corpus-roundtrip corpus-apijson","export export-examples","export-library export-settled","export-corpus","rest grpc"]' + echo 'go_shards=["runtime runtime-corpus runtime-exploration model","codegen migrate","cli translate","graph","smt","corpus corpus-sweep","export export-examples","export-library export-corpus","rest grpc"]' fi | tee -a "$GITHUB_OUTPUT" # The vendored copy is checked in this job, not static-and-integrity: the @@ -466,31 +466,19 @@ jobs: if: needs.changes.outputs.tier != 'quick' run: go run -C tools ./cmd/fuml-referee -check - # The RDF round-trip ratchet over every example on its own, so its per-verdict counts are legible and a - # skip must not pass; it is an examples/ sweep, so the queue tier takes the construct-cover subset here too. - - name: Run RDF corpus round-trip gate + # The RDF and api-json round-trip ratchets over every example on their own, so their per-verdict counts are + # legible and a skip must not pass; they are an examples/ sweep, so the queue tier takes the construct-cover subset here too. + - name: Run corpus round-trip gates if: needs.changes.outputs.tier != 'quick' env: OPENSYSML_CORPUS_SUBSET: ${{ needs.changes.outputs.tier == 'queue' && 'construct-cover' || '' }} run: | set -o pipefail - go test -count=1 -v ./tests/corpus -run 'TestCorpusRoundTrip' | tee corpus-roundtrip-gate.log - grep -E 'corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log + go test -count=1 -v ./tests/corpus -run 'TestExamplesSweep' | tee corpus-roundtrip-gate.log + grep -E ': corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log + grep -E 'api-json corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log if grep -qE '^\s*--- SKIP' corpus-roundtrip-gate.log; then - echo "error: the RDF corpus round-trip gate skipped" >&2 - exit 1 - fi - - - name: Run api-json corpus round-trip gate - if: needs.changes.outputs.tier != 'quick' - env: - OPENSYSML_CORPUS_SUBSET: ${{ needs.changes.outputs.tier == 'queue' && 'construct-cover' || '' }} - run: | - set -o pipefail - go test -count=1 -v ./tests/corpus -run 'TestCorpusAPIJSONRoundTrip' | tee api-json-roundtrip-gate.log - grep -E 'api-json corpus round trip: [0-9]+ files:' api-json-roundtrip-gate.log - if grep -qE '^\s*--- SKIP' api-json-roundtrip-gate.log; then - echo "error: the api-json corpus round-trip gate skipped" >&2 + echo "error: the corpus round-trip gates skipped" >&2 exit 1 fi diff --git a/.github/workflows/rebaseline.yml b/.github/workflows/rebaseline.yml index acf58dcf6c..f786c7edf9 100644 --- a/.github/workflows/rebaseline.yml +++ b/.github/workflows/rebaseline.yml @@ -187,7 +187,7 @@ jobs: - name: Re-record the per-file ratchets run: | - go test -count=1 ./tests/corpus -run 'TestPilotCorporaDiagnostics|TestSpecExamplesDiagnostics|TestPSSMSuiteMigration|TestCorpusRoundTrip|TestCorpusAPIJSONRoundTrip' \ + go test -count=1 ./tests/corpus -run 'TestPilotCorporaDiagnostics|TestSpecExamplesDiagnostics|TestPSSMSuiteMigration|TestExamplesSweep' \ -update-pilot-corpora -update-spec-examples -update-pssm-migration -update-corpus-roundtrip -update-api-json-roundtrip go test -count=1 ./internal/translate/export -run TestSymbolStructureMatchesGraphStructure -update-structure-differential go test -count=1 ./internal/exec/runtime -run TestLibraryAndPrefixSweep -update-library-prefix-sweep diff --git a/AGENTS.md b/AGENTS.md index e7c4402472..e66300b893 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ make build # build bin/sysml and bin/sysml-lsp (with version ldflags) make build-sysml # REPL binary only make build-lsp # LSP binary only make test # full suite: go test -race -coverprofile ... ./... -make test-shard SHARD=runtime # one CI shard of the race suite (runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-roundtrip|corpus-apijson|export|export-library|export-examples|export-corpus|export-settled|graph|codegen|translate|grpc|tools|rest|migrate|cli) +make test-shard SHARD=runtime # one CI shard of the race suite (runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-sweep|export|export-library|export-examples|export-corpus|graph|codegen|translate|grpc|tools|rest|migrate|cli) make test-quick SHARDS="rest grpc" # the same shards without -race, in turn: what a pull-request push runs make test-queue SHARDS="rest export-corpus" # the merge queue's tier: -race, but the examples/ sweeps (scripts/race-shard.sh --sweeps) race-free over the construct-cover subset make test-short # faster, no race detector @@ -102,10 +102,10 @@ answers under `internal/ir/docplan/markup/testdata/`, and the `Installed` test t against the pinned converter — re-record them (`asciidoctor-json.js < x.adoc > x.json`) when either moves. The RDF mapping has a per-file ratchet of its own over every model under `examples/`, the -downloaded corpora included: `TestCorpusRoundTrip` in `tests/corpus` converts each file -notation → Turtle → notation → Turtle and pins the verdict. Run it with both require variables -set after any change to `internal/translate/export`, adjudicate every movement, then regenerate with -`-update-corpus-roundtrip`. See `docs/project/rdf-corpus-roundtrip.md`. +downloaded corpora included: `TestExamplesSweep` in `tests/corpus` loads each file once, converts it +notation → Turtle → notation → Turtle (and the same through `api-json`) and pins the verdicts. Run it with +the require variables set after any change to `internal/translate/export`, adjudicate every movement, +then regenerate with `-update-corpus-roundtrip` (`-update-api-json-roundtrip`). See `docs/project/rdf-corpus-roundtrip.md`. --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bce12fa952..85e0ecd27b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -301,7 +301,7 @@ syntactic construct the whole corpus does; unset, they walk the whole corpus. **The quick tier** is what every push to a pull request gets, draft or ready, in about half the runner time of the full tier and a third of its wall time: the binaries, the same Go suite without -`-race` in ten groups of the race shards (`make test-quick`), the download-free static checks +`-race` in nine groups of the race shards (`make test-quick`), the download-free static checks (gofmt, vet, the production build, grpc imports, proto lint and breaking), the WebAssembly, Xpect and conformance gates, and only the clients, editors and docs whose own paths changed. The lint, the corpus and solver gates, the PDF toolchain and the fan-out from a service change to every @@ -309,7 +309,7 @@ client wait for the full tier. Only a push touching a path no area claims escala the full tier (`scripts/ci-changed-areas.sh`, and its test pins it). **The queue tier** runs on the merge queue's commit (below). It runs every job and gate of the -full tier — gofmt, `go vet`, `make lint`, every shard of the test suite (twenty, `make +full tier — gofmt, `go vet`, `make lint`, every shard of the test suite (eighteen, `make test-queue`), the binaries, the client suites, the conformance suite over each transport, the protobuf lint and wire-compatibility checks, the documentation hygiene checks (`make docs-check`, `make man-check` and the census check), the site build, the PDF toolchain, and in each client's diff --git a/Makefile b/Makefile index 238057cd39..7513268571 100644 --- a/Makefile +++ b/Makefile @@ -98,16 +98,14 @@ COVERAGE_PROFILE ?= coverage.txt COVERAGE_SHARD ?= # The coverage job's nodes: the race shards but the runtime's corpus ones, since without -race internal/exec/runtime # runs whole in about 19 minutes under -coverpkg; tests/export and tests/corpus whole overran 30 (see .circleci/config.yml). -COVERAGE_SHARDS := runtime model smt corpus corpus-roundtrip corpus-apijson export export-library export-examples export-corpus export-settled graph codegen translate grpc tools rest migrate cli +COVERAGE_SHARDS := runtime model smt corpus corpus-sweep export export-library export-examples export-corpus graph codegen translate grpc tools rest migrate cli # coverage-shard's splits of tests/corpus and tests/export by test name, test-shard's; empty for every other shard. -COVERAGE_SELECT_corpus = -skip '$(RACE_CORPUS_ROUNDTRIP)|$(RACE_CORPUS_APIJSON)' -COVERAGE_SELECT_corpus-roundtrip = -run '$(RACE_CORPUS_ROUNDTRIP)' -COVERAGE_SELECT_corpus-apijson = -run '$(RACE_CORPUS_APIJSON)' -COVERAGE_SELECT_export = -skip '$(RACE_EXPORT_CORPUS)|$(RACE_EXPORT_SETTLED)|$(RACE_EXPORT_LIBRARY)|$(RACE_EXPORT_EXAMPLES)' +COVERAGE_SELECT_corpus = -skip '$(RACE_CORPUS_SWEEP)' +COVERAGE_SELECT_corpus-sweep = -run '$(RACE_CORPUS_SWEEP)' +COVERAGE_SELECT_export = -skip '$(RACE_EXPORT_CORPUS)|$(RACE_EXPORT_LIBRARY)|$(RACE_EXPORT_EXAMPLES)' COVERAGE_SELECT_export-library = -run '$(RACE_EXPORT_LIBRARY)' COVERAGE_SELECT_export-examples = -run '$(RACE_EXPORT_EXAMPLES)' COVERAGE_SELECT_export-corpus = -run '$(RACE_EXPORT_CORPUS)' -COVERAGE_SELECT_export-settled = -run '$(RACE_EXPORT_SETTLED)' # The commands whose manual pages are generated and shipped, in section 1. COMMANDS := sysml sysml-lsp sysml-grpc sysml-jupyter-kernel sysml-mcp @@ -355,17 +353,15 @@ RACE_SHARD_SKIP := ^(TestTrainingExamplesSemanticErrors|TestCorpusGatesCacheStat # suite is as long as those four together, so runtime-exploration runs it as a shard of its own. RACE_RUNTIME_EXPLORATION := ^TestExploreWithIsExploreOverTheConformanceCorpus$$ RACE_RUNTIME_CORPUS := ^Test(ExploreWithIsExploreOverTheConformanceCorpus|CheckAgreesWithExploreOverTheConformanceCorpus|CheckWitnessesReplayOverTheConformanceCorpus|ExecutionConformance|ExecutionConformanceUnderPolicies)$$ -# The export-corpus and export-settled shards: the two tests/export suites that convert every example -# model, each about as long as the rest of the package under -race (the element form completes the family). +# The export-corpus shard: the tests/export suite that converts every example model, about as +# long as the rest of the package under -race. RACE_EXPORT_CORPUS := ^TestAPIJSONWritersNeverEmitNameReferences$$ -RACE_EXPORT_SETTLED := ^TestSettledCollectionsAreWhatReconcilingWouldFind$$ # tests/export's two parallel per-file sweeps: 60 and 27 of the remainder's 103 CPU-minutes under -race (37 minutes wall), so a shard each. RACE_EXPORT_LIBRARY := ^TestLibraryFilesComeBackFromTheGraphAlone$$ RACE_EXPORT_EXAMPLES := ^TestExamplesCorpusInvariants$$ -# The corpus-roundtrip and corpus-apijson shards: tests/corpus's two round trips over every example -# model, about 20 minutes each under -race, one after the other; corpus is the rest of the package. -RACE_CORPUS_ROUNDTRIP := ^TestCorpusRoundTrip$$ -RACE_CORPUS_APIJSON := ^TestCorpusAPIJSONRoundTrip$$ +# The corpus-sweep shard: tests/corpus's one load of every example model, on which it checks the +# two round trips, settled collections and the gRPC agreement; corpus is the rest of the package. +RACE_CORPUS_SWEEP := ^TestExamplesSweep$$ RACE_SHARD_TOOLS_SKIP := ^(TestSuiteRead|TestSuiteClassification|TestEmitSuite|TestSuiteClassificationReasons|TestSuiteLibraryCallsAreClassified|TestSuiteReadsControlAndObjectFlow|TestSuiteReadsClassifiers|TestSuiteReadsExceptionHandlers)$$ # test-shard's -run/-skip per shard: the by-test-name shards and the package shards they are cut @@ -373,38 +369,36 @@ RACE_SHARD_TOOLS_SKIP := ^(TestSuiteRead|TestSuiteClassification|TestEmitSuite|T SHARD_SELECT_runtime-exploration = -run '$(RACE_RUNTIME_EXPLORATION)' -skip '$(RACE_SHARD_SKIP)' SHARD_SELECT_runtime-corpus = -run '$(RACE_RUNTIME_CORPUS)' -skip '$(RACE_SHARD_SKIP)|$(RACE_RUNTIME_EXPLORATION)' SHARD_SELECT_runtime = -skip '$(RACE_SHARD_SKIP)|$(RACE_RUNTIME_CORPUS)' -SHARD_SELECT_corpus-roundtrip = -run '$(RACE_CORPUS_ROUNDTRIP)' -skip '$(RACE_SHARD_SKIP)' -SHARD_SELECT_corpus-apijson = -run '$(RACE_CORPUS_APIJSON)' -skip '$(RACE_SHARD_SKIP)' -SHARD_SELECT_corpus = -skip '$(RACE_SHARD_SKIP)|$(RACE_CORPUS_ROUNDTRIP)|$(RACE_CORPUS_APIJSON)' +SHARD_SELECT_corpus-sweep = -run '$(RACE_CORPUS_SWEEP)' -skip '$(RACE_SHARD_SKIP)' +SHARD_SELECT_corpus = -skip '$(RACE_SHARD_SKIP)|$(RACE_CORPUS_SWEEP)' SHARD_SELECT_export-corpus = -run '$(RACE_EXPORT_CORPUS)' -skip '$(RACE_SHARD_SKIP)' -SHARD_SELECT_export-settled = -run '$(RACE_EXPORT_SETTLED)' -skip '$(RACE_SHARD_SKIP)' SHARD_SELECT_export-library = -run '$(RACE_EXPORT_LIBRARY)' -skip '$(RACE_SHARD_SKIP)' SHARD_SELECT_export-examples = -run '$(RACE_EXPORT_EXAMPLES)' -skip '$(RACE_SHARD_SKIP)' -SHARD_SELECT_export = -skip '$(RACE_SHARD_SKIP)|$(RACE_EXPORT_CORPUS)|$(RACE_EXPORT_SETTLED)|$(RACE_EXPORT_LIBRARY)|$(RACE_EXPORT_EXAMPLES)' +SHARD_SELECT_export = -skip '$(RACE_SHARD_SKIP)|$(RACE_EXPORT_CORPUS)|$(RACE_EXPORT_LIBRARY)|$(RACE_EXPORT_EXAMPLES)' SHARD_SELECT = $(or $(SHARD_SELECT_$(SHARD)),-skip '$(RACE_SHARD_SKIP)') # -race for the race suite (test-shard); test-quick runs the same shards with it empty. TEST_SHARD_RACE ?= -race -test-shard: stdlib-snapshot ## Run one CI shard of the race suite (SHARD=runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-roundtrip|corpus-apijson|export|export-library|export-examples|export-corpus|export-settled|graph|codegen|translate|grpc|tools|rest|migrate|cli) +test-shard: stdlib-snapshot ## Run one CI shard of the race suite (SHARD=runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-sweep|export|export-library|export-examples|export-corpus|graph|codegen|translate|grpc|tools|rest|migrate|cli) @echo "Running Go $(if $(TEST_SHARD_RACE),race ,)tests, shard $(SHARD)$(if $(OPENSYSML_CORPUS_SUBSET), over the $(OPENSYSML_CORPUS_SUBSET) corpus subset,)..." @# 55m per package; its job's ceiling is 60. internal/exec/runtime (31-57 minutes whole under -race), tests/export - @# (past 55; five shards) and tests/corpus (about 40; three) run split by test name above. + @# (past 55; four shards) and tests/corpus (about 40; two) run split by test name above. pkgs=$$(scripts/race-shard.sh $(SHARD)) && go test $(SHARD_SELECT) -v $(TEST_SHARD_RACE) -pgo=off -timeout 55m $$pkgs @# The tools module runs with tools (tests/wasm) on CircleCI and, as the PR workflow has no @# tools shard (tests/wasm is its WebAssembly gate job there), with rest on GitHub Actions. case "$(SHARD)" in rest|tools) go test -C $(TOOLS_DIR) -skip '$(RACE_SHARD_TOOLS_SKIP)' -v $(TEST_SHARD_RACE) -pgo=off -timeout 55m ./... ;; esac -test-quick: stdlib-snapshot ## Run the race-free quick tier over one or more race shards, in turn (SHARDS="graph" | "export-corpus export-settled" | "rest grpc" | ...) +test-quick: stdlib-snapshot ## Run the race-free quick tier over one or more race shards, in turn (SHARDS="graph" | "export-corpus export-library" | "rest grpc" | ...) @echo "Running Go tests without race detection, shards $(SHARDS)..." @# Every pull-request push runs this instead of test-shard: the same shards, selections and skips, no -race. - @test -n "$(SHARDS)" || { echo "Error: set SHARDS to one or more of runtime runtime-corpus runtime-exploration model smt corpus corpus-roundtrip corpus-apijson export export-library export-examples export-corpus export-settled graph codegen translate grpc tools rest migrate cli"; exit 1; } + @test -n "$(SHARDS)" || { echo "Error: set SHARDS to one or more of runtime runtime-corpus runtime-exploration model smt corpus corpus-sweep export export-library export-examples export-corpus graph codegen translate grpc tools rest migrate cli"; exit 1; } for shard in $(SHARDS); do $(MAKE) --no-print-directory test-shard SHARD=$$shard TEST_SHARD_RACE= || exit 1; done test-queue: stdlib-snapshot ## Run the merge queue's tier over one or more race shards, in turn (SHARDS="export-corpus" | "rest grpc" | ...) @echo "Running the queue tier, shards $(SHARDS)..." @# The merge queue runs this: test-shard with -race on every shard but the examples/ sweeps @# (scripts/race-shard.sh --sweeps), which run without -race over the construct-cover subset. - @test -n "$(SHARDS)" || { echo "Error: set SHARDS to one or more of runtime runtime-corpus runtime-exploration model smt corpus corpus-roundtrip corpus-apijson export export-library export-examples export-corpus export-settled graph codegen translate grpc tools rest migrate cli"; exit 1; } + @test -n "$(SHARDS)" || { echo "Error: set SHARDS to one or more of runtime runtime-corpus runtime-exploration model smt corpus corpus-sweep export export-library export-examples export-corpus graph codegen translate grpc tools rest migrate cli"; exit 1; } sweeps=" $$(scripts/race-shard.sh --sweeps | tr '\n' ' ')" && for shard in $(SHARDS); do \ case "$$sweeps" in \ *" $$shard "*) OPENSYSML_CORPUS_SUBSET=construct-cover $(MAKE) --no-print-directory test-shard SHARD=$$shard TEST_SHARD_RACE= ;; \ diff --git a/changes/unreleased/load-once-check-many.changed.md b/changes/unreleased/load-once-check-many.changed.md new file mode 100644 index 0000000000..63cd008881 --- /dev/null +++ b/changes/unreleased/load-once-check-many.changed.md @@ -0,0 +1 @@ +- The per-model checks over `examples/` — the Turtle and api-json round-trip ratchets, settled collections, the gRPC element-properties agreement and the without-derived read-back — run on one load of each model in `TestExamplesSweep` (`tests/corpus`), which replaces `TestCorpusRoundTrip`, `TestCorpusAPIJSONRoundTrip`, `TestSettledCollectionsAreWhatReconcilingWouldFind`, `TestElementPropertiesAgreeWithTheAPIJSONExport` and `TestWithoutDerivedReadsBackEveryExample`; the race shards `corpus-roundtrip`, `corpus-apijson` and `export-settled` become `corpus-sweep`, and `scripts/race-shard.sh --sweeps` names `corpus-sweep` in place of the three. The round-trip expectation files keep every verdict; only their regenerate command line changed. The sweep iterates `OPENSYSML_CORPUS_SUBSET` as the tests it replaced did, a directory whole when any of its files is listed. diff --git a/docs/internals/execution-ir-design.md b/docs/internals/execution-ir-design.md index f63f9e451c..a83230826d 100644 --- a/docs/internals/execution-ir-design.md +++ b/docs/internals/execution-ir-design.md @@ -504,7 +504,7 @@ full suite with `OPENSYSML_REQUIRE_TRAINING_CORPUS=1 OPENSYSML_REQUIRE_PILOT_COR OPENSYSML_REQUIRE_PILOT_LIBRARY_XMI=1`, the `tools` module, `make docs-counts`, and `TestPackageLayering`. The baselines that must not move at **any** step: the pilot Xpect, differential and rejection ratchets (`tests/corpus`, `TestPilotCorpora`), the RDF round trip -(`TestCorpusRoundTrip`), the fUML and PSSM referee records under `tools/referee`, the execution +(`TestExamplesSweep`), the fUML and PSSM referee records under `tools/referee`, the execution conformance suite (194 fixture files under `internal/exec/runtime/testdata/conformance`, `TestExecutionConformance` and `…UnderPolicies`), the golden traces (`TestExecutionTrace`, which print names, not pointers, and so are the right pin for an identity change), every diff --git a/docs/project/construct-cover-subset.md b/docs/project/construct-cover-subset.md index 61341b6c1f..2ca67c49bd 100644 --- a/docs/project/construct-cover-subset.md +++ b/docs/project/construct-cover-subset.md @@ -61,7 +61,7 @@ and `FilterPaths` answer "iterate this file?". A file outside `examples/` — th loaded as a batch against loaded serially, or by the CLI against the workspace) iterates a directory when any of its files is listed, and loads the whole directory: a partial directory would change what the test measures. -- The per-file ratchets — `TestCorpusRoundTrip`, `TestCorpusAPIJSONRoundTrip`, the pilot and +- The per-file ratchets — both round trips of `TestExamplesSweep`, the pilot and specification-example gates — compare the visited files' verdicts against the expectation file and leave the unvisited files' recorded lines unchecked; the recorded root sizes are still checked against the whole corpus, so a partial download fails as before. The pilot and diff --git a/docs/project/rdf-corpus-roundtrip.md b/docs/project/rdf-corpus-roundtrip.md index 59f43cf54d..b20ed36219 100644 --- a/docs/project/rdf-corpus-roundtrip.md +++ b/docs/project/rdf-corpus-roundtrip.md @@ -20,11 +20,12 @@ pin in `scripts/pilot-pin.sh`, and the textual examples of the two specification | `spec-examples/sysml` | 341 | | **total** | **876** | -**Gate:** `TestCorpusRoundTrip` in `tests/corpus/roundtrip_test.go` converts each file +**Gate:** `TestExamplesSweep` in `tests/corpus/examples_sweep_test.go` loads each file once (the +verdicts and expectation files are `tests/corpus/roundtrip_test.go`'s) and converts it notation → Turtle (hop 1) → notation → Turtle (hop 2) and records one verdict per file in `tests/corpus/testdata/corpus_roundtrip_expected.txt`, so a file whose verdict moves in either direction, or that appears or disappears, fails the test -**Regenerate:** `go test ./tests/corpus -run TestCorpusRoundTrip -update-corpus-roundtrip` +**Regenerate:** `go test ./tests/corpus -run TestExamplesSweep -update-corpus-roundtrip` **Required in CI:** `OPENSYSML_REQUIRE_TRAINING_CORPUS=1`, `OPENSYSML_REQUIRE_PILOT_CORPORA=1` and `OPENSYSML_REQUIRE_SPEC_EXAMPLES=1` in both `.circleci/config.yml` and `.github/workflows/pr.yml`, under which an absent or empty downloaded root fails instead of skipping. Both configurations also run the gate on its own so @@ -163,15 +164,15 @@ adjudicated: ## The API element form -The same file walk, verdicts and policy run a second time over the API's JSON element form +The same verdicts and policy run over the API's JSON element form on the same load of each file (`api-json`, [rdf-mapping.md § The API element form](../reference/rdf-mapping.md#the-api-element-form)): -`TestCorpusAPIJSONRoundTrip` converts each file notation → `api-json` (hop 1) → notation → +`TestExamplesSweep` converts each file notation → `api-json` (hop 1) → notation → `api-json` (hop 2) and pins the verdict in `tests/corpus/testdata/api_json_roundtrip_expected.txt`. `whitespace-only` compares the two JSON documents as the graphs `export.ReadAPIJSON` reads them to, under the same `sysx:sourceText` normalisation. -**Regenerate:** `go test ./tests/corpus -run TestCorpusAPIJSONRoundTrip -update-api-json-roundtrip` -**Required in CI:** the same two variables, run as its own step beside the Turtle gate. +**Regenerate:** `go test ./tests/corpus -run TestExamplesSweep -update-api-json-roundtrip` +**Required in CI:** the same two variables, in the same step as the Turtle gate. | Verdict | Files | |---|---| @@ -199,6 +200,7 @@ verdict is worse than its Turtle verdict is a defect in `export/api_json_out.go` `stable` verdict here does not prove the graph could be written back without the text. That is what the `sysx:sourceText`-stripping tests in `export_test.go` are for; see `.agents/skills/testing-rdf-roundtrip/SKILL.md`. -- **It does not run when the downloaded corpora are absent**, except in CI, where the require - variables make absence a failure. Locally the skip is announced with a `GATE NOT RUN` banner on - stderr and the fetch command to run. +- **The ratchets are not checked when a downloaded corpus is absent**, except in CI, where the + require variables make absence a failure. Locally the absence is announced with a `GATE NOT RUN` + banner on stderr and the fetch command to run; the sweep's other properties still run over the + models present. diff --git a/docs/project/roadmap.md b/docs/project/roadmap.md index 554da39365..cdfea2e923 100644 --- a/docs/project/roadmap.md +++ b/docs/project/roadmap.md @@ -220,10 +220,10 @@ A tag cannot be cut over a corpus regression. Pull requests run one CI, the GitH workflow `.github/workflows/pr.yml`. Its `changes` job decides the tier — *quick* for a push to a pull request, draft or ready; *full* for a merge-queue commit, a pull request labelled `ci:full`, or a push touching a package where `-race` has found bugs (`scripts/ci-changed-areas.sh`) — and which of the -client, editor and documentation jobs the diff touches; the Go suite runs sharded (twenty race +client, editor and documentation jobs the diff touches; the Go suite runs sharded (eighteen race shards in the full tier — `runtime`, `runtime-corpus`, `runtime-exploration`, `model`, `smt`, `corpus`, -`corpus-roundtrip`, `corpus-apijson`, `export`, `export-library`, `export-examples`, `export-corpus`, `export-settled`, `graph`, `codegen`, -`translate`, `grpc`, `rest`, `migrate` and `cli`, #1440 — and ten race-free groups of them in the quick tier) with every corpus +`corpus-sweep`, `export`, `export-library`, `export-examples`, `export-corpus`, `graph`, `codegen`, +`translate`, `grpc`, `rest`, `migrate` and `cli`, #1440 — and nine race-free groups of them in the quick tier) with every corpus required — `OPENSYSML_REQUIRE_TRAINING_CORPUS`, `_PILOT_CORPORA`, `_SPEC_EXAMPLES`, `_PSSM_SUITE`, `_SYSML_V1TOV2`, `_FUML_SUITE`, `_PILOT_LIBRARY_XMI`, `_SMT`, `_WASM` and `_REFERENCE_FMUS` set to `1`, the LLM-benchmark and PDF toolchains required where their jobs run — beside the diff --git a/docs/project/spec-compliance/02-detailed-semantic-compliance-map/25-structural-interface-and-analysis-notation.md b/docs/project/spec-compliance/02-detailed-semantic-compliance-map/25-structural-interface-and-analysis-notation.md index a77dabe3f7..48064521f6 100644 --- a/docs/project/spec-compliance/02-detailed-semantic-compliance-map/25-structural-interface-and-analysis-notation.md +++ b/docs/project/spec-compliance/02-detailed-semantic-compliance-map/25-structural-interface-and-analysis-notation.md @@ -37,7 +37,7 @@ semantics layer over the conjugation parity of the typing/specialization chain. | A requirement-like body admits usage elements, so a connector, flow or message written with its kind keyword (`connection connect r to x;`) is a member of a `requirement`, `constraint`, `concern`, `objective`, `use case` or `view` body exactly as it is at package level (§8.2.2.19 `RequirementBody` → `DefinitionBodyItem` → `UsageElement`) | `parser/behavior.go` `parseRequirementMember`, `usageIsSubstantive` (a connector or flow usage declares no name, so its ends are what make it substantive) | `parse/requirement_body_prefixed_usages.golden` | ✅ Faithful | | An indexed connector end, `connect s.y#(1) to k.u` — also in `connection … connect`, `interface`, `allocate`/`allocation`, `bind`/`binding`, a structural `flow` and KerML's `connector`/`binding of` — attaches one element of the collection-valued feature `s.y` (1-based, as `#(` indexes everywhere). The end's type is the element type and its multiplicity one, so it conforms to a connection definition's `end source[1] : P` as `connect s.y to k.u` would with `s.y : P[1]`; the end still reference-subsets the whole feature `s.y`, and the `ConnectorEnd`/`ownedEndFeature` reflection is unchanged. Not standard SysML v2: a `ConnectorEnd` is a feature chain (§8.2.2.13.1) and `#(` is an expression operator (KerML §8.2.5.8.2), so the standard has no spelling for it — the pinned pilot rejects it as a syntax error and OpenSysML reports it as `nonstandard-notation` (an error under `-strict`). Scope: the index is admitted on the last chain segment only (`a.b#(1).c` stays the single expression-end error) and not on a succession or transition end, a named end's `::>`/`references` target or an action-body pin flow, which lowers to an action graph that addresses whole pins. A `send` routed through a connection with an indexed end reaches the port the end selects from, as it would through the whole-feature end: the runtime delivers a message to a port of an object, not to one element of the port | `parser/defusage.go` `parseEndIndex`, `parseFeatureEnd`, `parseFlowEnd`, `admitsIndexedEnd`; `ast/defusage.go` `ConnectorEnd.AttachedTarget`, `AttachedIndex`, `AttachedSelection`, `EndSelection`; `semantics/connector.go` `ConnectorEndAttachment.Index`/`Selection`, `FlowEndAttachment.Index`; `semantics/end_multiplicity.go` `ConnectorEndMultiplicityIsOne`; `passes/nonstandard_notation.go` `notationWalker.indexedEnds` | `parser/negative_test.go` `TestIndexedEndParsesAsAnEnd`, `TestExpressionEndIsReportedOnce`; `parse/indexed_connector_ends.sysml`; `semantics/indexed_connector_end_test.go`; `passes/indexed_connector_end_test.go` `TestIndexedConnectorEndIsAnExtension`, `TestWholeFeatureEndsAndIndexExpressionsAreSilent`; `passes/nonstandard_notation_strict_test.go` | ⚠️ tool-defined extension | | The index of an indexed end is an Integer-valued expression; a constant index below 1 selects no element and one above the upper bound of the multiplicity governing the feature — the one it declares, or inherits by redefinition without restating (`port :>> y` over `Base::y : P[2]`) — is outside it (`s.y#(3)` on `port y : P[2]`), each a type error at the index. An index known only at run time is checked when the connector materializes: an out-of-range or non-Integer index, or an index into a feature holding no value, is a typed `connectorEndError` (wrapping `ErrIndexOutOfRange`, `errUninitializedFeatureValue`) that materializes nothing and never panics. A binding end selecting an element keeps the path of the feature it selects from, so the binding is found from that feature as from the other end: `bind gauge = tank.levels#(2)` gives `gauge` the element, while a binding between two selected elements (`bind xs#(1) = ys#(1)`) checks that the elements agree — unequal ones are a `bindingConflictError` read from either feature — and that the connector's and the ends' stated multiplicities admit the one value it identifies (`binding [2] bind xs#(1) = ys#(1)` is a multiplicity violation), and assigns through neither index, each feature keeping its own value | `passes/typecheck_expr.go` `exprChecker.checkIndexedEnds`, `checkEndIndex` (over `Model.GoverningMultiplicityOf`); `runtime/connector.go` `attachConnectorEnd` (evaluates the selection through the `#(` evaluator), `endText`; `runtime/binding.go` `bindingEndpoint.selects`, `Context.selectedBindingElement`, `attemptBinding`; `lower/binding.go` `BindingEnd.Index` | `passes/indexed_connector_end_test.go` `TestIndexedConnectorEndIndexIsTyped`, `TestIndexedConnectorEndIndexIsBoundByTheGoverningMultiplicity`; `runtime/robustness_indexed_connector_end_test.go` `TestRuntimeRobustnessIndexedConnectorEnd`, `TestRuntimeRobustnessIndexedBindingBetweenSelectedElements`; `lower/binding_test.go` `TestToBindingsKeepsTheFeatureAnIndexedEndSelectsFrom`; `conformance/indexed_binding_end_selects_element`, `conformance/indexed_connection_end_selects_port_element` | ✅ faithful | -| An indexed end is written back as `#( index )` by the formatter and the notation converters, and the RDF/API JSON mapping states the index as `sysx:endElement`, an expression node beside the standard `ReferenceSubsetting` of the whole feature, so a reader of the standard vocabulary sees `s.y` connected and the decoder rebuilds the end from the graph alone | `export/rdf_out.go` `connectorEndSpec.element`, `xEndElement`; `export/end_forms.go` `decoder.endElementText` | `tests/export/indexed_connector_ends_test.go` `TestIndexedConnectorEndsCarryTheirIndexAsSysxEndElement`, `TestIndexedConnectorEndsRoundTripThroughAPIJSON`; `tests/corpus` `TestCorpusRoundTrip`, `TestCorpusAPIJSONRoundTrip` (unchanged: no OMG corpus writes the notation) | ✅ faithful | +| An indexed end is written back as `#( index )` by the formatter and the notation converters, and the RDF/API JSON mapping states the index as `sysx:endElement`, an expression node beside the standard `ReferenceSubsetting` of the whole feature, so a reader of the standard vocabulary sees `s.y` connected and the decoder rebuilds the end from the graph alone | `export/rdf_out.go` `connectorEndSpec.element`, `xEndElement`; `export/end_forms.go` `decoder.endElementText` | `tests/export/indexed_connector_ends_test.go` `TestIndexedConnectorEndsCarryTheirIndexAsSysxEndElement`, `TestIndexedConnectorEndsRoundTripThroughAPIJSON`; `tests/corpus` `TestExamplesSweep` (unchanged: no OMG corpus writes the notation) | ✅ faithful | **Known limitations of this notation** diff --git a/docs/project/spec-compliance/06-model-persistence-and-rdf-interchange.md b/docs/project/spec-compliance/06-model-persistence-and-rdf-interchange.md index 117952e235..32ffd4f142 100644 --- a/docs/project/spec-compliance/06-model-persistence-and-rdf-interchange.md +++ b/docs/project/spec-compliance/06-model-persistence-and-rdf-interchange.md @@ -31,7 +31,7 @@ a whole model or that its vocabulary is settled — see | An expression's `referent`/`targetFeature` link is the `memberElement` of a `sysml:Membership` the expression owns (`_referent`, `_preferent`, `_targetFeature`), listed under `ownedRelationship`/`ownedMembership` where the notation places it, before the result parameter | `export/rdf_expr.go` `featureReferent`, `expressionOperands` (`ast/expr.go` `FeatureChainExpr`) | `export_test.go:TestFixtureElementIDsRoundTrip`, `export/api_json_import_test.go`, `tests/export/result_expression_test.go:TestExpressionBodyParameterIsDeclaredOutsideTheBody` | ✅ Faithful | | sysml-toolkit interchange JSON imports both ways: `convert --to compact-json` and `--to full-json` decode to byte-identical notation, the toolkit's root `Namespace`+`OwningMembership` wrapper is transparent, `{"@ref": }` and `unresolved:`-derived targets read as the name they spell, stated defaults on `isImpliedIncluded` elements collapse back, and ends the notation cannot place are refused | `export/rdf_in_toolkit.go`, `export/api_json_in.go` `ReadAPIJSON` | `export/api_json_import_test.go` (`tests/export/testdata/interchange/` fixtures), `:TestUUIDRoundTrip` | ✅ Faithful | | `-id uuid` mints name-based uuids the way the library convention does (root: `uuid5(NamespaceURL, elementIRI(root))`; derived: `uuid5(pkg, `), declared and normative ids are never re-derived, and the form reads back with its derived ids implied — no `@ElementId` annotations | `export/id_form.go` `IDUUID`, `export/rdf_identity.go` `subjectOf`, `export/id_form.go` `minted`/`mintedNode`, `export/verbatim.go` `demoteStale` | `export/api_json_import_test.go:TestUUIDIDs`, `:TestUUIDRoundTrip` (byte-identical to the default-form round trip), `:TestUUIDDeclaredElementID` | ✅ Faithful | -| Round trip through the element form pinned per file over `examples/` | `tests/corpus/roundtrip_test.go` `TestCorpusAPIJSONRoundTrip`, baseline `testdata/api_json_roundtrip_expected.txt` | the committed models, the downloaded corpora and the specification examples pinned per file, with `stable`, `whitespace-only` (whitespace inside `sysx:sourceText` literals), `unwritable`, `refused:syntax` (specification examples that are fragments, refused before conversion) and `graph-diff` verdicts, counted in `testdata/api_json_roundtrip_expected.txt` | ✅ Faithful | +| Round trip through the element form pinned per file over `examples/` | `tests/corpus/examples_sweep_test.go` `TestExamplesSweep`, baseline `testdata/api_json_roundtrip_expected.txt` | the committed models, the downloaded corpora and the specification examples pinned per file, with `stable`, `whitespace-only` (whitespace inside `sysx:sourceText` literals), `unwritable`, `refused:syntax` (specification examples that are fragments, refused before conversion) and `graph-diff` verdicts, counted in `testdata/api_json_roundtrip_expected.txt` | ✅ Faithful | | The element form is what the SysML v2 API's own commit path accepts | `tools/referee/flexo` `Measure`, the `api-json-commit` side posts the emitted elements as `DataVersion` payloads | `TestFlexoInterop` (opt-in, `.agents/skills/flexo-interop`), `interop_expected.txt` | ✅ Faithful — the form's 78 elements (the root `Namespace` and its `OwningMembership` included) and 717 of its 878 properties are served back, and the Turtle graph-load path delivers its 76 elements and 702 of 863 the same way, every standard property included; the 161 lost are the `sysx:` properties, which the service drops on both paths | | `individual` on a definition (`OccurrenceDefinition::isIndividual`, SysML v2 §8.3.9.3) carried structurally, so the modifier survives a round trip with `sysx:sourceText` stripped for every definition kind the parser lets it prefix (`individual part def`, `individual item def`, `individual occurrence def`, `individual action def`, …) and a usage typed by such a definition still validates; an `individual def` carries the same flag and reads back by its kind keyword alone; a definition without the modifier carries no flag | `export/rdf_out.go` `encode` (the definition's `flags`, the same `sysml:isIndividual` a usage writes), `export/rdf_in.go` `definitionHead` | `export_test.go:TestFixturesComeBackFromTheGraphAlone`, `:TestGoldenConversions`, `:TestRoundTripIsLossless` (fixture `testdata/convert/individual_definitions.sysml`), `model/export_roundtrip_test.go:TestNotationFromTheGraphAloneAnalysesLikeTheOriginal` (the notation written from the graph alone analyses exactly as the original) | ✅ Faithful | | Deterministic, reversible element IRIs keyed by qualified name, with ids in `[A-Za-z0-9_-]+` | `rdf/vocab.go` `ElementIRI`, `rdf/ids.go` `EncodeElementID`/`DecodeElementID` | `rdf_test.go:TestElementIRIRoundTrip`, `rdf/ids_test.go`, `export_test.go:TestElementIRIsEncodeQualifiedNames`, `TestFixtureElementIDsRoundTrip` | ✅ Faithful | diff --git a/docs/reference/rdf-mapping.md b/docs/reference/rdf-mapping.md index 3c6b296f11..e772e7a399 100644 --- a/docs/reference/rdf-mapping.md +++ b/docs/reference/rdf-mapping.md @@ -2540,7 +2540,7 @@ any graph this mapping holds: derivation stays a declared `@ElementId` annotation. The per-file ratchet over `examples/` runs for this form too: -`TestCorpusAPIJSONRoundTrip` in `tests/corpus/roundtrip_test.go` converts each +`TestExamplesSweep` in `tests/corpus` (verdicts in `roundtrip_test.go`) converts each model notation → `api-json` → notation → `api-json` and pins the verdict in `testdata/api_json_roundtrip_expected.txt` ([rdf-corpus-roundtrip.md](../project/rdf-corpus-roundtrip.md)). Its verdicts diff --git a/internal/translate/interop/modelsync/modelsync.go b/internal/translate/interop/modelsync/modelsync.go index 6cbddcff49..05ee844f0a 100644 --- a/internal/translate/interop/modelsync/modelsync.go +++ b/internal/translate/interop/modelsync/modelsync.go @@ -247,7 +247,6 @@ func isOwnerPredicate(predicate string) bool { // graph rather than recomputing — a collection's order among them — so a // publish without derived properties keeps them: a project written without // them could not be loaded back as notation. -// TestWithoutDerivedReadsBackEveryExample pins the list against the reader. var readerDerived = map[string]bool{} func init() { diff --git a/internal/translate/interop/modelsync/modelsync_test.go b/internal/translate/interop/modelsync/modelsync_test.go index 1a5937ad15..a79cb7989f 100644 --- a/internal/translate/interop/modelsync/modelsync_test.go +++ b/internal/translate/interop/modelsync/modelsync_test.go @@ -1,9 +1,6 @@ package modelsync import ( - "bytes" - "os" - "path/filepath" "strings" "testing" @@ -11,7 +8,6 @@ import ( "github.com/Open-MBEE/OpenSysML/internal/translate/export" "github.com/Open-MBEE/OpenSysML/internal/translate/interop/reposync" "github.com/Open-MBEE/OpenSysML/internal/translate/rdf" - "github.com/Open-MBEE/OpenSysML/tests/testutil/corpussubset" ) const model = `package Vehicles { @@ -126,53 +122,3 @@ func TestWithoutDerivedDropsDerivedPropertiesTheReaderRecomputes(t *testing.T) { } } } - -// TestWithoutDerivedReadsBackEveryExample pins readerDerived against the -// notation reader: every example publishes without derived properties to the -// notation it publishes with them. -func TestWithoutDerivedReadsBackEveryExample(t *testing.T) { - files, err := filepath.Glob("../../../../examples/*.sysml") - if err != nil || len(files) == 0 { - t.Fatalf("no examples: %v", err) - } - files = corpussubset.Must(t, "../../../..").FilterPaths("../../../../examples", files) - roundTrip := func(graph *rdf.Graph) ([]byte, error) { - data, err := export.WriteAPIJSON(graph) - if err != nil { - return nil, err - } - back, err := export.ReadAPIJSON(data) - if err != nil { - return nil, err - } - return Notation(back, nil) - } - dropped := 0 - for _, file := range files { - data, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - graph, err := convert.SysMLToRDF(file, data) - if err != nil { - continue - } - want, err := roundTrip(graph) - if err != nil { - continue - } - cut := WithoutDerived(graph) - dropped += graph.Len() - cut.Len() - got, err := roundTrip(cut) - if err != nil { - t.Errorf("%s: without derived properties: %v", filepath.Base(file), err) - continue - } - if !bytes.Equal(got, want) { - t.Errorf("%s: the notation differs without derived properties", filepath.Base(file)) - } - } - if dropped == 0 { - t.Error("no example has a derived property to leave out") - } -} diff --git a/scripts/race-shard.sh b/scripts/race-shard.sh index 76d28344ed..e77e1baaf0 100755 --- a/scripts/race-shard.sh +++ b/scripts/race-shard.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # Prints the root-module package paths assigned to one race-test shard. -# Usage: scripts/race-shard.sh runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-roundtrip|corpus-apijson|export|export-library|export-examples|export-corpus|export-settled|graph|codegen|translate|grpc|tools|rest|migrate|cli | --sweeps +# Usage: scripts/race-shard.sh runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-sweep|export|export-library|export-examples|export-corpus|graph|codegen|translate|grpc|tools|rest|migrate|cli | --sweeps + # A hyphenated shard is a by-test-name part of its package (runtime, corpus and export are each split # that way in make test-shard); the other shards are whole packages, rest everything not named here. set -euo pipefail cd "$(dirname "$0")/.." -shards='runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-roundtrip|corpus-apijson|export|export-library|export-examples|export-corpus|export-settled|graph|codegen|translate|grpc|tools|rest|migrate|cli' +shards='runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-sweep|export|export-library|export-examples|export-corpus|graph|codegen|translate|grpc|tools|rest|migrate|cli' # The shards whose one test walks every example model, which the queue tier (make test-queue) runs race-free over the subset; # export-library walks the bundled libraries, and grpc's and translate's sweeps share a shard with race-prone tests: all keep -race. -sweeps='export-corpus export-settled export-examples corpus-roundtrip corpus-apijson' +sweeps='corpus-sweep export-corpus export-examples' if [[ $# -eq 1 && $1 == --sweeps ]]; then printf '%s\n' $sweeps @@ -25,7 +26,7 @@ fi shard=$1 module=github.com/Open-MBEE/OpenSysML runtime="$module/internal/exec/runtime" -# internal/workspace/model (~26 min), tests/corpus (~39 min: its two round trips ~20 each) and +# internal/workspace/model (~26 min), tests/corpus (~39 min, its examples sweep apart) and # internal/exec/smt (~17 min) shared one node that ran 41 minutes under -race, so each is its own. workspace_model="$module/internal/workspace/model" corpus="$module/tests/corpus" @@ -55,9 +56,9 @@ cli="$module/cmd/sysml" named_packages=("$runtime" "$workspace_model" "$corpus" "$smt" "$export" "$model" "$translate" "$grpc_tests" "$grpc_frontend" "$passes" "$migrate" "$codegen" "$wasm" "$cli") case "$shard" in - runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-roundtrip|corpus-apijson|export|export-library|export-examples|export-corpus|export-settled|graph|codegen|translate|grpc|tools|rest|migrate|cli) ;; + runtime|runtime-corpus|runtime-exploration|model|smt|corpus|corpus-sweep|export|export-library|export-examples|export-corpus|graph|codegen|translate|grpc|tools|rest|migrate|cli) ;; *) - echo "usage: scripts/race-shard.sh $shards" >&2 + echo "usage: scripts/race-shard.sh $shards | --sweeps" >&2 exit 2 ;; esac @@ -80,10 +81,10 @@ case "$shard" in smt) printf '%s\n' "$smt" ;; - corpus|corpus-roundtrip|corpus-apijson) + corpus|corpus-sweep) printf '%s\n' "$corpus" ;; - export|export-library|export-examples|export-corpus|export-settled) + export|export-library|export-examples|export-corpus) printf '%s\n' "$export" ;; graph) diff --git a/tests/corpus/construct_cover_test.go b/tests/corpus/construct_cover_test.go index aedc76b849..77e1aa50ae 100644 --- a/tests/corpus/construct_cover_test.go +++ b/tests/corpus/construct_cover_test.go @@ -22,8 +22,11 @@ const constructCoverList = "testdata/construct_cover.txt" // syntactic feature any model under examples/ exhibits. func TestConstructCover(t *testing.T) { // Parsing only, it sweeps the whole corpus whatever OPENSYSML_CORPUS_SUBSET - // says; the roots skip or are required as in the other corpus gates. - files, _ := corpusRoundTripFiles(t) + // says; an absent root skips it, or fails it where the root is required. + files, _, whole := corpusRoundTripFiles(t) + if !whole { + t.Skip("a downloaded corpus is absent, so the subset cannot be checked against the whole") + } features := make(map[string][]string, len(files)) for _, rel := range files { data, err := os.ReadFile(filepath.Join(corpusRoundTripExamples, filepath.FromSlash(rel))) diff --git a/tests/corpus/examples_sweep_test.go b/tests/corpus/examples_sweep_test.go new file mode 100644 index 0000000000..d3880c246c --- /dev/null +++ b/tests/corpus/examples_sweep_test.go @@ -0,0 +1,586 @@ +package corpus + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + pb "github.com/Open-MBEE/OpenSysML/api/proto" + "github.com/Open-MBEE/OpenSysML/internal/frontend/grpc" + "github.com/Open-MBEE/OpenSysML/internal/syntax/parser" + "github.com/Open-MBEE/OpenSysML/internal/syntax/source" + "github.com/Open-MBEE/OpenSysML/internal/translate/convert" + "github.com/Open-MBEE/OpenSysML/internal/translate/export" + "github.com/Open-MBEE/OpenSysML/internal/translate/interop/modelsync" + "github.com/Open-MBEE/OpenSysML/internal/translate/rdf" + "github.com/Open-MBEE/OpenSysML/tests/testutil/corpussubset" +) + +// sweepFixtures holds the export fixtures the settled-collections property +// covers besides the examples. +const sweepFixtures = "../export/testdata" + +// sweepModel is one model of the sweep, loaded once: read, parsed, and exported +// through convert's single-document path, with that graph written both ways. +type sweepModel struct { + rel string // slash path relative to examples/, or the fixture's path + path string + fixture bool // an export fixture: settled collections only + wanted bool // iterated by the subset; otherwise only parsed, for its directory's model + + doc export.ModelDocument + parsed bool // no parser diagnostic at all + graph *rdf.Graph + graphErr error // convert's hop-1 refusal: a syntax error, or the export's + turtle []byte + apiJSON []byte + apiJSONErr error +} + +// load reads, parses and exports the model as convert.Convert's hop 1 does. +func (m *sweepModel) load() error { + src, err := os.ReadFile(m.path) + if err != nil { + return err + } + file := source.New(m.rel, src) + p := parser.New(file) + root := p.ParseFile() + m.doc = export.ModelDocument{File: file, Root: root} + m.parsed = len(p.Diagnostics) == 0 + if !m.wanted { + return nil + } + if syntax := convert.SyntaxErrorOf(m.rel, file, p.Diagnostics); syntax != nil { + m.graphErr = syntax + return nil + } + m.graph, m.graphErr = export.ToRDFWith(file, root, export.IDQualifiedName) + if m.graphErr != nil { + return nil + } + m.turtle = rdf.WriteTurtle(m.graph) + m.apiJSON, m.apiJSONErr = export.WriteAPIJSON(m.graph) + return nil +} + +// apiJSONHop1Err is what convert.Convert to api-json would have refused with. +func (m *sweepModel) apiJSONHop1Err() error { + if m.graphErr != nil { + return m.graphErr + } + return m.apiJSONErr +} + +// settledCandidate says whether the model takes the settled-collections property: +// a SysML document that parsed clean, as tests/export's sweep selected them. +func (m *sweepModel) settledCandidate() bool { + return m.parsed && strings.HasSuffix(m.path, ".sysml") +} + +// sweepTally is what one model added to the sweep's per-property counts. +type sweepTally struct { + settled, agreementExamples, converted, elements, pairs, dropped int +} + +// sweepCounts are the per-property tallies the sweep logs and asserts on. +type sweepCounts struct { + mu sync.Mutex + total sweepTally +} + +func (c *sweepCounts) add(tally sweepTally) { + c.mu.Lock() + defer c.mu.Unlock() + c.total.settled += tally.settled + c.total.agreementExamples += tally.agreementExamples + c.total.converted += tally.converted + c.total.elements += tally.elements + c.total.pairs += tally.pairs + c.total.dropped += tally.dropped +} + +// sweepDirectory is one directory's SysML documents, checked together as one +// model once every one of them has been loaded. +type sweepDirectory struct { + models []*sweepModel + pending atomic.Int32 + broken atomic.Bool // a document did not parse clean, so the directory is not a model +} + +// TestExamplesSweep loads every model under examples/ once and checks every per-model property +// on that load: both round-trip ratchets, settled collections, the gRPC agreement, modelsync's read-back. +func TestExamplesSweep(t *testing.T) { + subset := corpussubset.Must(t, "../..") + corpus, totals, ratchet := corpusRoundTripFiles(t) + if !ratchet && (*updateCorpusRoundTrip || *updateAPIJSONRoundTrip) { + t.Fatal("re-recording the round-trip expectations needs every corpus downloaded") + } + files := subset.Filter(corpus) + models := make([]*sweepModel, 0, len(corpus)) + for _, rel := range corpus { + models = append(models, &sweepModel{rel: rel, path: filepath.Join(corpusRoundTripExamples, filepath.FromSlash(rel)), wanted: subset.Wants(rel)}) + } + models = append(models, sweepFixtureModels(t)...) + models = sweepSubsetModels(models) + directories := sweepDirectoriesOf(models) + if !subset.Whole() { + t.Logf("%s=%s: %d of %d examples, %d models", corpussubset.EnvVar, subset.Name(), len(files), len(corpus), len(models)) + } + + // An empty semantic cache, as on a fresh checkout and in CI. + t.Setenv("XDG_CACHE_HOME", t.TempDir()) + + turtle := make([]string, len(models)) + apiJSON := make([]string, len(models)) + failures := make([]error, len(models)) + var counts sweepCounts + ctx := context.Background() + + // A directory is checked as soon as its last document is loaded, in one of the slots the + // per-model workers load in: -parallel bounds the models in hand, a whole-directory model among them. + workers := sweepSlots() + slots := make(chan struct{}, workers) + for range workers { + slots <- struct{}{} + } + ready := make(chan *sweepDirectory, len(directories)) + var checking sync.WaitGroup + checking.Add(1) + go func() { + defer checking.Done() + for d := range ready { + <-slots + settled := d.check(t) + slots <- struct{}{} + if settled { + counts.add(sweepTally{settled: 1}) + } + } + }() + + next := make(chan int) + var loading sync.WaitGroup + for w := 0; w < workers; w++ { + // Each worker's own service, caching the one model it has in hand: a shared LRU could + // evict a worker's model between its RPCs, and a larger one would hold models already checked. + srv, err := grpc.NewService(1, "test") + if err != nil { + t.Fatal(err) + } + loading.Add(1) + go func() { + defer loading.Done() + for i := range next { + m := models[i] + <-slots + if err := m.load(); err != nil { + failures[i] = err + } else if m.wanted { + if ratchet && !m.fixture { + turtle[i], failures[i] = corpusRoundTripVerdict(m.rel, m.turtle, m.graphErr) + if failures[i] == nil { + apiJSON[i], failures[i] = apiJSONRoundTripVerdict(m.rel, m.apiJSON, m.apiJSONHop1Err()) + } + } + counts.add(sweepProperties(t, ctx, srv, m)) + } + slots <- struct{}{} + if !strings.HasSuffix(m.path, ".sysml") { + m.doc = export.ModelDocument{} + continue + } + d := directories[filepath.Dir(m.path)] + if !m.parsed { + d.broken.Store(true) + } + if d.pending.Add(-1) == 0 { + ready <- d + } + } + }() + } + for _, i := range sweepOrder(models, directories) { + next <- i + } + close(next) + loading.Wait() + close(ready) + checking.Wait() + + gotTurtle := make(map[string]string, len(files)) + gotAPIJSON := make(map[string]string, len(files)) + for i, m := range models { + switch { + case failures[i] != nil: + t.Errorf("%s: %v", m.rel, failures[i]) + case m.wanted && !m.fixture: + gotTurtle[m.rel] = turtle[i] + gotAPIJSON[m.rel] = apiJSON[i] + } + } + + total := counts.total + t.Logf("%d models checked", total.settled) + if total.settled < 50 { + t.Errorf("only %d models converted; the fixtures moved", total.settled) + } + if total.converted == 0 || total.elements == 0 { + t.Errorf("no example converts to api-json (%d models, %d elements)", total.converted, total.elements) + } + t.Logf("%d of %d examples convert; %d elements, %d (element, property) pairs agree", + total.converted, total.agreementExamples, total.elements, total.pairs) + if total.dropped == 0 { + t.Error("no example has a derived property to leave out") + } + + if !ratchet { + t.Log("round-trip ratchets not checked: a corpus is absent") + return + } + corpusRoundTripGate.check(t, subset, corpus, files, totals, gotTurtle) + apiJSONRoundTripGate.check(t, subset, corpus, files, totals, gotAPIJSON) +} + +// sweepSubsetModels keeps the models the subset iterates and, as tests/export's sweep did, every +// SysML document of a directory one of them is in, so the directory is still checked whole. +func sweepSubsetModels(models []*sweepModel) []*sweepModel { + directories := map[string]bool{} + for _, m := range models { + if m.wanted && strings.HasSuffix(m.path, ".sysml") { + directories[filepath.Dir(m.path)] = true + } + } + kept := models[:0] + for _, m := range models { + if m.wanted || (strings.HasSuffix(m.path, ".sysml") && directories[filepath.Dir(m.path)]) { + kept = append(kept, m) + } + } + return kept +} + +// sweepProperties runs the per-model properties on one loaded model (failures go to t, safe from a +// worker); settled collections come last, over their own export, once the hop-1 graph is released. +func sweepProperties(t *testing.T, ctx context.Context, srv *grpc.Service, m *sweepModel) (tally sweepTally) { + if !m.fixture { + // The downloaded pilot corpora have gates of their own. + if !strings.HasPrefix(m.rel, "pilot-corpora/") { + tally.agreementExamples = 1 + if m.apiJSONHop1Err() == nil { + tally.elements, tally.pairs = elementPropertiesAgree(t, ctx, srv, m) + tally.converted = 1 + } + } + if m.graphErr == nil && !strings.Contains(m.rel, "/") && strings.HasSuffix(m.rel, ".sysml") { + var err error + if tally.dropped, err = withoutDerivedReadsBack(m.graph); err != nil { + t.Errorf("%s: %v", m.rel, err) + } + } + } + m.graph, m.turtle, m.apiJSON = nil, nil, nil + if m.settledCandidate() { + // The settled property is over the model path, which vouches for its collections. + if graph, err := export.ModelToRDFWith([]export.ModelDocument{m.doc}, export.IDQualifiedName); err == nil { + if err := settledCollectionsHold(graph); err != nil { + t.Errorf("%s: %v", m.rel, err) + } else { + tally.settled = 1 + } + } + } + return tally +} + +// sweepFixtureModels are the SysML export fixtures, which only the +// settled-collections property covers. +func sweepFixtureModels(t *testing.T) []*sweepModel { + t.Helper() + var models []*sweepModel + err := filepath.WalkDir(sweepFixtures, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() && strings.HasSuffix(path, ".sysml") { + models = append(models, &sweepModel{rel: filepath.ToSlash(path), path: path, fixture: true, wanted: true}) + } + return nil + }) + if err != nil { + t.Fatalf("scan %s: %v", sweepFixtures, err) + } + sort.Slice(models, func(i, j int) bool { return models[i].rel < models[j].rel }) + return models +} + +// sweepSlots is how many models the sweep has in hand at once: -parallel's value, as it bounded +// the parallel subtests of tests/export's sweep, so a CI class sizes the memory peak the same way. +func sweepSlots() int { + if f := flag.Lookup("test.parallel"); f != nil { + if n, err := strconv.Atoi(f.Value.String()); err == nil && n > 0 { + return n + } + } + return runtime.GOMAXPROCS(0) +} + +// sweepDirectoriesOf groups the SysML models by directory, each group to be +// checked as one model once its documents are loaded. +func sweepDirectoriesOf(models []*sweepModel) map[string]*sweepDirectory { + directories := map[string]*sweepDirectory{} + for _, m := range models { + if !strings.HasSuffix(m.path, ".sysml") { + continue + } + dir := filepath.Dir(m.path) + d := directories[dir] + if d == nil { + d = &sweepDirectory{} + directories[dir] = d + } + d.models = append(d.models, m) + d.pending.Add(1) + } + return directories +} + +// sweepOrder loads the largest directories first, so their whole-directory checks overlap +// the remaining per-model work instead of running alone at the end. +func sweepOrder(models []*sweepModel, directories map[string]*sweepDirectory) []int { + order := make([]int, len(models)) + for i := range order { + order[i] = i + } + size := func(i int) int { + if d := directories[filepath.Dir(models[i].path)]; d != nil { + return len(d.models) + } + return 0 + } + sort.SliceStable(order, func(a, b int) bool { + if sa, sb := size(order[a]), size(order[b]); sa != sb { + return sa > sb + } + return filepath.Dir(models[order[a]].path) < filepath.Dir(models[order[b]].path) + }) + return order +} + +// check runs the settled-collections property over the directory's documents as one model +// (two or more, every one parsed clean) and releases them; reports whether a model was checked. +func (d *sweepDirectory) check(t *testing.T) bool { + defer func() { + for _, m := range d.models { + m.doc = export.ModelDocument{} + } + }() + if len(d.models) < 2 || d.broken.Load() { + return false + } + names := make([]string, len(d.models)) + docs := make([]export.ModelDocument, len(d.models)) + for i, m := range d.models { + names[i], docs[i] = m.rel, m.doc + } + graph, err := export.ModelToRDFWith(docs, export.IDQualifiedName) + if err != nil { + return false // a model the export refuses is not converted at all + } + if err := settledCollectionsHold(graph); err != nil { + t.Errorf("%s: %v", strings.Join(names, ","), err) + return false + } + return true +} + +// settledCollectionsHold: the encoder's graph says its collections are settled, a copy that does +// not vouch for itself reconciles to itself unchanged, and both write the same api-json bytes. +func settledCollectionsHold(graph *rdf.Graph) error { + if !graph.CollectionsSettled() { + return errors.New("the encoder's graph does not say its collections are settled") + } + unvouched := rdf.NewGraphOf(graph.Triples(), graph.Prefixes) + reconciled, err := rdf.ReconcileCollections(unvouched) + if err != nil { + return fmt.Errorf("reconciling the encoder's collections: %w", err) + } + if reconciled != unvouched { + return errors.New("reconciling rewrote the encoder's collections, so they were not settled") + } + fast, err := export.WriteAPIJSON(graph) + if err != nil { + return err + } + checkedOut, err := export.WriteAPIJSON(unvouched) + if err != nil { + return err + } + if !bytes.Equal(fast, checkedOut) { + return errors.New("api-json differs between the settled graph and the checked one") + } + return nil +} + +// elementPropertiesAgree checks GetElementProperties answers what the api-json export writes for +// every (element, metamodel property) pair of the model (`sysx:` keys aside); returns the counts. +func elementPropertiesAgree(t *testing.T, ctx context.Context, srv *grpc.Service, m *sweepModel) (elements, pairs int) { + absolute, err := filepath.Abs(m.path) + if err != nil { + t.Errorf("%s: %v", m.rel, err) + return + } + parsed, err := srv.ParseFile(ctx, &pb.ParseFileRequest{Source: &pb.ParseFileRequest_FilePath{FilePath: absolute}}) + if err != nil { + t.Errorf("%s: ParseFile: %v", m.rel, err) + return + } + var objects []map[string]any + decoder := json.NewDecoder(bytes.NewReader(m.apiJSON)) + decoder.UseNumber() + if err := decoder.Decode(&objects); err != nil { + t.Errorf("%s: api-json: %v", m.rel, err) + return + } + for _, object := range objects { + id, _ := object["@id"].(string) + resp, err := srv.GetElementProperties(ctx, &pb.GetElementPropertiesRequest{ModelHash: parsed.ModelHash, ElementId: id}) + if err != nil { + t.Errorf("%s: GetElementProperties(%s): %v", m.rel, id, err) + return + } + if resp.Error != "" { + t.Errorf("%s: GetElementProperties(%s): %s", m.rel, id, resp.Error) + continue + } + if metaclass, _ := object["@type"].(string); resp.Element.Metaclass != metaclass { + t.Errorf("%s: %s is a %s in the export and a %s in the answer", m.rel, id, metaclass, resp.Element.Metaclass) + } + elements++ + answered := map[string]*pb.ElementProperty{} + for _, property := range resp.Properties { + answered[property.Name] = property + } + for name, value := range object { + if name == "@id" || name == "@type" || strings.HasPrefix(name, "sysx:") { + continue + } + pairs++ + property, ok := answered[name] + if !ok { + t.Errorf("%s: %s.%s is exported but not answered", m.rel, id, name) + continue + } + delete(answered, name) + if want, got := apiJSONSpelling(value), propertySpelling(property); want != got { + t.Errorf("%s: %s.%s exported as %s, answered as %s", m.rel, id, name, want, got) + } + } + for name := range answered { + t.Errorf("%s: %s.%s is answered but not exported", m.rel, id, name) + } + } + return elements, pairs +} + +// apiJSONSpelling flattens an exported property value to one comparable text: +// element references by id, literals by value, in collection order. +func apiJSONSpelling(value any) string { + switch v := value.(type) { + case nil: + return "" + case []any: + parts := make([]string, 0, len(v)) + for _, member := range v { + parts = append(parts, apiJSONSpelling(member)) + } + return strings.Join(parts, " | ") + case map[string]any: + id, _ := v["@id"].(string) + return "element " + id + case json.Number: + return numberSpelling(v.String()) + case string: + return "string " + v + case bool: + return "bool " + strconv.FormatBool(v) + } + return fmt.Sprintf("%T %v", value, value) +} + +// propertySpelling flattens an answered property the way apiJSONSpelling does the export. +func propertySpelling(property *pb.ElementProperty) string { + if property.Unsupported != "" { + return "unsupported: " + property.Unsupported + } + parts := make([]string, 0, len(property.Values)) + for _, value := range property.Values { + switch v := value.Value.(type) { + case *pb.ElementPropertyValue_Element: + parts = append(parts, "element "+v.Element.Id) + case *pb.ElementPropertyValue_StringValue: + parts = append(parts, "string "+v.StringValue) + case *pb.ElementPropertyValue_BoolValue: + parts = append(parts, "bool "+strconv.FormatBool(v.BoolValue)) + case *pb.ElementPropertyValue_IntValue: + parts = append(parts, numberSpelling(strconv.FormatInt(v.IntValue, 10))) + case *pb.ElementPropertyValue_RealValue: + parts = append(parts, numberSpelling(strconv.FormatFloat(v.RealValue, 'g', -1, 64))) + case *pb.ElementPropertyValue_Number: + parts = append(parts, numberSpelling(v.Number)) + default: + parts = append(parts, fmt.Sprintf("%T", value.Value)) + } + } + return strings.Join(parts, " | ") +} + +// numberSpelling normalizes a decimal text so 2.0 and 2 compare equal. +func numberSpelling(text string) string { + if f, err := strconv.ParseFloat(text, 64); err == nil { + return "number " + strconv.FormatFloat(f, 'g', -1, 64) + } + return "number " + text +} + +// withoutDerivedReadsBack pins modelsync's readerDerived against the notation reader: the model +// publishes without derived properties to the notation it publishes with them; returns the triples left out. +func withoutDerivedReadsBack(graph *rdf.Graph) (dropped int, err error) { + roundTrip := func(graph *rdf.Graph) ([]byte, error) { + data, err := export.WriteAPIJSON(graph) + if err != nil { + return nil, err + } + back, err := export.ReadAPIJSON(data) + if err != nil { + return nil, err + } + return modelsync.Notation(back, nil) + } + want, err := roundTrip(graph) + if err != nil { + return 0, nil // a model the publisher cannot read back is not compared + } + cut := modelsync.WithoutDerived(graph) + dropped = graph.Len() - cut.Len() + got, err := roundTrip(cut) + if err != nil { + return dropped, fmt.Errorf("without derived properties: %w", err) + } + if !bytes.Equal(got, want) { + return dropped, errors.New("the notation differs without derived properties") + } + return dropped, nil +} diff --git a/tests/corpus/roundtrip_test.go b/tests/corpus/roundtrip_test.go index 1d6bf44809..918f0fa9ec 100644 --- a/tests/corpus/roundtrip_test.go +++ b/tests/corpus/roundtrip_test.go @@ -8,11 +8,9 @@ import ( "os" "path/filepath" "regexp" - "runtime" "slices" "sort" "strings" - "sync" "testing" "github.com/Open-MBEE/OpenSysML/internal/syntax/source" @@ -44,7 +42,7 @@ const ( "# no longer converts) and refused: (notation -> Turtle refused). This\n" + "# is a per-file ratchet, not a claim that any verdict is right; see\n" + "# docs/project/rdf-corpus-roundtrip.md. Regenerate with:\n" + - "# go test ./tests/corpus -run TestCorpusRoundTrip -update-corpus-roundtrip\n" + "# go test ./tests/corpus -run TestExamplesSweep -update-corpus-roundtrip\n" apiJSONRoundTripHeader = "# Round-trip verdict for every model under examples/, as \"\\t\":\n" + "# notation -> the API's JSON element form (hop 1) -> notation -> the JSON\n" + @@ -56,19 +54,17 @@ const ( "# converts) and refused: (notation -> the JSON form refused).\n" + "# This is a per-file ratchet, not a claim that any verdict is right;\n" + "# see docs/project/rdf-corpus-roundtrip.md. Regenerate with:\n" + - "# go test ./tests/corpus -run TestCorpusAPIJSONRoundTrip -update-api-json-roundtrip\n" + "# go test ./tests/corpus -run TestExamplesSweep -update-api-json-roundtrip\n" ) -// roundTripGate is one per-file ratchet over the corpus: the same walk, -// worker pool, expectation file and comparison, differing only in the hop -// format each verdict round-trips through. +// roundTripGate is one per-file ratchet over the corpus: an expectation file and its comparison, +// differing only in the hop format; TestExamplesSweep measures both on one load of each model. type roundTripGate struct { name string // the gate's name in the summary line CI greps updateFlag string // the flag regeneration messages name update *bool expected string header string - verdict func(rel string, src []byte) (string, error) } var corpusRoundTripGate = roundTripGate{ @@ -77,7 +73,6 @@ var corpusRoundTripGate = roundTripGate{ update: updateCorpusRoundTrip, expected: corpusRoundTripExpected, header: corpusRoundTripHeader, - verdict: corpusRoundTripVerdict, } var apiJSONRoundTripGate = roundTripGate{ @@ -86,7 +81,6 @@ var apiJSONRoundTripGate = roundTripGate{ update: updateAPIJSONRoundTrip, expected: apiJSONRoundTripExpected, header: apiJSONRoundTripHeader, - verdict: apiJSONRoundTripVerdict, } // corpusRoundTripRoot is one downloaded corpus under examples/. The committed @@ -132,21 +126,20 @@ var corpusRoundTripRoots = []corpusRoundTripRoot{ }, } -// skip mirrors corpusGate.skip: skip locally, fail when the require-env is -// set, and announce it on stderr either way. -func (r corpusRoundTripRoot) skip(t *testing.T, reason string) { +// absent mirrors corpusGate.skip for one ratchet root: fail when the require-env is set, +// announce on stderr otherwise. The sweep's other properties still run over what is present. +func (r corpusRoundTripRoot) absent(t *testing.T, reason string) { t.Helper() hint := fmt.Sprintf("examples/%s not downloaded (run %s)", r.name, r.fetch) if os.Getenv(r.requireEnv) != "" { t.Fatalf("%s=%s but %s: %s", r.requireEnv, os.Getenv(r.requireEnv), reason, hint) } - fmt.Fprintf(os.Stderr, "\n!!! GATE NOT RUN: %s SKIPPED - %s.\n"+ - "!!! The corpus is absent, so this run proves nothing about it.\n"+ + fmt.Fprintf(os.Stderr, "\n!!! GATE NOT RUN: the round-trip ratchets of %s SKIPPED - %s.\n"+ + "!!! The corpus is absent, so this run proves nothing about it; %s\n"+ "!!! Fetch it with %s and re-run\n"+ - "!!! go test -count=1 ./tests/corpus -run TestCorpusRoundTrip\n"+ + "!!! go test -count=1 ./tests/corpus -run TestExamplesSweep\n"+ "!!! CI sets %s=1, where an absent corpus fails instead of skipping.\n\n", - t.Name(), reason, r.fetch, r.requireEnv) - t.Skip(hint) + t.Name(), reason, hint, r.fetch, r.requireEnv) } // corpusRoundTripRootOf names the root a path relative to examples/ belongs to. @@ -159,19 +152,20 @@ func corpusRoundTripRootOf(rel string) string { return corpusRoundTripCommitted } -// corpusRoundTripFiles returns every model under examples/ as sorted -// slash-separated relative paths, plus the file count per root. -func corpusRoundTripFiles(t *testing.T) ([]string, map[string]int) { +// corpusRoundTripFiles returns every model under examples/ as sorted slash-separated relative +// paths, the file count per root, and whether every root is present so the ratchets can be checked. +func corpusRoundTripFiles(t *testing.T) (files []string, totals map[string]int, ratchet bool) { t.Helper() + missing := map[string]bool{} for _, root := range corpusRoundTripRoots { dir := filepath.Join(corpusRoundTripExamples, filepath.FromSlash(root.name)) if _, err := os.Stat(dir); os.IsNotExist(err) { - root.skip(t, dir+" is missing") + root.absent(t, dir+" is missing") + missing[root.name] = true } } - var files []string err := filepath.WalkDir(corpusRoundTripExamples, func(path string, entry os.DirEntry, err error) error { if err != nil { return err @@ -196,7 +190,7 @@ func corpusRoundTripFiles(t *testing.T) ([]string, map[string]int) { } sort.Strings(files) - totals := make(map[string]int, len(corpusRoundTripRoots)+1) + totals = make(map[string]int, len(corpusRoundTripRoots)+1) for _, rel := range files { totals[corpusRoundTripRootOf(rel)]++ } @@ -205,19 +199,19 @@ func corpusRoundTripFiles(t *testing.T) ([]string, map[string]int) { } // An empty directory is a broken download or a bad cache restore, not a corpus. for _, root := range corpusRoundTripRoots { - if totals[root.name] == 0 { - root.skip(t, "examples/"+root.name+" holds no model files") + if !missing[root.name] && totals[root.name] == 0 { + root.absent(t, "examples/"+root.name+" holds no model files") + missing[root.name] = true } } - return files, totals + return files, totals, len(missing) == 0 } -// corpusRoundTripVerdict classifies one model's notation -> Turtle -> notation -// -> Turtle trip. An error means the harness failed, never a verdict. -func corpusRoundTripVerdict(rel string, src []byte) (string, error) { - hop1, err := convert.Convert(rel, src, convert.FormatSysML, convert.FormatTurtle) - if err != nil { - return "refused:" + refusalClass(rel, err), nil +// corpusRoundTripVerdict classifies one model's notation -> Turtle -> notation -> Turtle trip from +// its hop 1 (the Turtle written, or the refusal). An error means the harness failed, never a verdict. +func corpusRoundTripVerdict(rel string, hop1 []byte, hop1Err error) (string, error) { + if hop1Err != nil { + return "refused:" + refusalClass(rel, hop1Err), nil } back, err := convert.Convert(rel+".ttl", hop1, convert.FormatTurtle, convert.FormatSysML) if err != nil { @@ -244,13 +238,11 @@ func corpusRoundTripVerdict(rel string, src []byte) (string, error) { return "graph-diff", nil } -// apiJSONRoundTripVerdict classifies one model's notation -> api-json -> -// notation -> api-json trip. The two hops parse back to graphs so the -// whitespace-only check compares what the JSON states, not its key order. -func apiJSONRoundTripVerdict(rel string, src []byte) (string, error) { - hop1, err := convert.Convert(rel, src, convert.FormatSysML, convert.FormatAPIJSON) - if err != nil { - return "refused:" + refusalClass(rel, err), nil +// apiJSONRoundTripVerdict classifies notation -> api-json -> notation -> api-json from its hop 1; +// the hops parse back to graphs so whitespace-only compares what the JSON states, not key order. +func apiJSONRoundTripVerdict(rel string, hop1 []byte, hop1Err error) (string, error) { + if hop1Err != nil { + return "refused:" + refusalClass(rel, hop1Err), nil } back, err := convert.Convert(rel+".json", hop1, convert.FormatAPIJSON, convert.FormatSysML) if err != nil { @@ -366,47 +358,6 @@ func refusalClass(name string, err error) string { } } -// verdicts measures every file on a worker pool, indexed like files so -// scheduling never affects the order. -func (g roundTripGate) verdicts(t *testing.T, files []string) map[string]string { - t.Helper() - - verdicts := make([]string, len(files)) - failures := make([]error, len(files)) - next := make(chan int) - var wg sync.WaitGroup - for w := 0; w < runtime.GOMAXPROCS(0); w++ { - wg.Add(1) - go func() { - defer wg.Done() - for i := range next { - rel := files[i] - src, err := os.ReadFile(filepath.Join(corpusRoundTripExamples, filepath.FromSlash(rel))) - if err != nil { - failures[i] = err - continue - } - verdicts[i], failures[i] = g.verdict(rel, src) - } - }() - } - for i := range files { - next <- i - } - close(next) - wg.Wait() - - got := make(map[string]string, len(files)) - for i, rel := range files { - if failures[i] != nil { - t.Errorf("%s: %v", rel, failures[i]) - continue - } - got[rel] = verdicts[i] - } - return got -} - // summary is the per-verdict count line CI greps for. func (g roundTripGate) summary(files []string, got map[string]string) string { counts := make(map[string]int) @@ -467,17 +418,10 @@ func (g roundTripGate) writeExpected(t *testing.T, files []string, got map[strin } } -// run measures every file and either regenerates the gate's expectation or -// fails on any movement from it. -func (g roundTripGate) run(t *testing.T) { - subset := corpussubset.Must(t, "../..") - corpus, totals := corpusRoundTripFiles(t) - files := subset.Filter(corpus) - - // An empty semantic cache, as on a fresh checkout and in CI. - t.Setenv("XDG_CACHE_HOME", t.TempDir()) - - got := g.verdicts(t, files) +// check takes the sweep's verdict of every visited file (files, the subset of corpus) and either +// regenerates the gate's expectation or fails on any movement from it. +func (g roundTripGate) check(t *testing.T, subset corpussubset.Subset, corpus, files []string, totals map[string]int, got map[string]string) { + t.Helper() t.Log(g.summary(files, got)) if *g.update { @@ -522,19 +466,6 @@ func (g roundTripGate) run(t *testing.T) { } } -// TestCorpusRoundTrip is a per-file ratchet on the RDF round trip of every -// model under examples/; any movement fails it. See docs/project/rdf-corpus-roundtrip.md. -func TestCorpusRoundTrip(t *testing.T) { - corpusRoundTripGate.run(t) -} - -// TestCorpusAPIJSONRoundTrip is the same ratchet over the API's JSON element -// form, whose reader and writer share the RDF graph. See -// docs/project/rdf-corpus-roundtrip.md. -func TestCorpusAPIJSONRoundTrip(t *testing.T) { - apiJSONRoundTripGate.run(t) -} - // Refusal classes must carry no location or identifier, however spelt. func TestRefusalClass(t *testing.T) { cases := []struct { diff --git a/tests/corpus/testdata/api_json_roundtrip_expected.txt b/tests/corpus/testdata/api_json_roundtrip_expected.txt index eb63fb3a9c..2a33206cea 100644 --- a/tests/corpus/testdata/api_json_roundtrip_expected.txt +++ b/tests/corpus/testdata/api_json_roundtrip_expected.txt @@ -8,7 +8,7 @@ # converts) and refused: (notation -> the JSON form refused). # This is a per-file ratchet, not a claim that any verdict is right; # see docs/project/rdf-corpus-roundtrip.md. Regenerate with: -# go test ./tests/corpus -run TestCorpusAPIJSONRoundTrip -update-api-json-roundtrip +# go test ./tests/corpus -run TestExamplesSweep -update-api-json-roundtrip stable action-executor-demo.sysml stable analysis-demo/lander.sysml stable analysis-results-demo/lander-results.sysml diff --git a/tests/corpus/testdata/corpus_roundtrip_expected.txt b/tests/corpus/testdata/corpus_roundtrip_expected.txt index 9112ec58e8..9de8bac093 100644 --- a/tests/corpus/testdata/corpus_roundtrip_expected.txt +++ b/tests/corpus/testdata/corpus_roundtrip_expected.txt @@ -7,7 +7,7 @@ # no longer converts) and refused: (notation -> Turtle refused). This # is a per-file ratchet, not a claim that any verdict is right; see # docs/project/rdf-corpus-roundtrip.md. Regenerate with: -# go test ./tests/corpus -run TestCorpusRoundTrip -update-corpus-roundtrip +# go test ./tests/corpus -run TestExamplesSweep -update-corpus-roundtrip stable action-executor-demo.sysml stable analysis-demo/lander.sysml stable analysis-results-demo/lander-results.sysml diff --git a/tests/export/settled_collections_test.go b/tests/export/settled_collections_test.go deleted file mode 100644 index 37267745ab..0000000000 --- a/tests/export/settled_collections_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package export_test - -import ( - "bytes" - "os" - "path/filepath" - "sort" - "strings" - "sync/atomic" - "testing" - - "github.com/Open-MBEE/OpenSysML/internal/syntax/parser" - "github.com/Open-MBEE/OpenSysML/internal/syntax/source" - "github.com/Open-MBEE/OpenSysML/internal/translate/export" - "github.com/Open-MBEE/OpenSysML/internal/translate/rdf" - "github.com/Open-MBEE/OpenSysML/tests/testutil/corpussubset" -) - -// The encoder's graph says its collections are settled, so writing api-json -// skips ReconcileCollections and reads each collection from its typed triples -// rather than parsing its JSON annotation. This holds the claim to the check it -// replaces: over every model the repository's examples and export fixtures hold, -// one document at a time and each directory's documents together, a copy of the -// graph that does not vouch for itself reconciles to itself unchanged, and -// writes the same api-json byte for byte. -func TestSettledCollectionsAreWhatReconcilingWouldFind(t *testing.T) { - models := settledModels(t) - if len(models) == 0 { - t.Fatal("no models found") - } - var checked atomic.Int64 - t.Run("models", func(t *testing.T) { - for _, files := range models { - name := strings.Join(files, ",") - t.Run(name, func(t *testing.T) { - t.Parallel() - docs, ok := parsedDocuments(files) - if !ok { - return - } - graph, err := export.ModelToRDFWith(docs, export.IDQualifiedName) - if err != nil { - return // a model the export refuses is not converted at all - } - if !graph.CollectionsSettled() { - t.Fatalf("%s: the encoder's graph does not say its collections are settled", name) - } - unvouched := rdf.NewGraphOf(graph.Triples(), graph.Prefixes) - reconciled, err := rdf.ReconcileCollections(unvouched) - if err != nil { - t.Fatalf("%s: reconciling the encoder's collections: %v", name, err) - } - if reconciled != unvouched { - t.Fatalf("%s: reconciling rewrote the encoder's collections, so they were not settled", name) - } - fast, err := export.WriteAPIJSON(graph) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - checkedOut, err := export.WriteAPIJSON(unvouched) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if !bytes.Equal(fast, checkedOut) { - t.Errorf("%s: api-json differs between the settled graph and the checked one", name) - } - checked.Add(1) - }) - } - }) - t.Logf("%d models checked", checked.Load()) - if checked.Load() < 50 { - t.Errorf("only %d models converted; the fixtures moved", checked.Load()) - } -} - -// settledModels are the SysML models under the examples and the export -// fixtures: each file alone, and each directory's files together. -func settledModels(t *testing.T) [][]string { - t.Helper() - subset := corpussubset.Must(t, "../..") - byDir := map[string][]string{} - for _, root := range []string{"../../examples", "testdata"} { - _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err == nil && !d.IsDir() && strings.HasSuffix(path, ".sysml") { - byDir[filepath.Dir(path)] = append(byDir[filepath.Dir(path)], path) - } - return nil - }) - } - dirs := make([]string, 0, len(byDir)) - for dir := range byDir { - dirs = append(dirs, dir) - } - sort.Strings(dirs) - var models [][]string - for _, dir := range dirs { - files := byDir[dir] - sort.Strings(files) - for _, file := range files { - if subset.WantsPath("../../examples", file) { - models = append(models, []string{file}) - } - } - if len(files) > 1 && subset.WantsAny("../../examples", files) { - models = append(models, files) - } - } - return models -} - -// parsedDocuments parses files as one model, reporting false when one does not parse clean. -func parsedDocuments(files []string) ([]export.ModelDocument, bool) { - docs := make([]export.ModelDocument, 0, len(files)) - for _, path := range files { - content, err := os.ReadFile(path) - if err != nil { - return nil, false - } - file := source.New(path, content) - p := parser.New(file) - root := p.ParseFile() - if len(p.Diagnostics) > 0 { - return nil, false - } - docs = append(docs, export.ModelDocument{File: file, Root: root}) - } - return docs, true -} diff --git a/tests/grpc/element_properties_agreement_test.go b/tests/grpc/element_properties_agreement_test.go deleted file mode 100644 index 24c1bdc104..0000000000 --- a/tests/grpc/element_properties_agreement_test.go +++ /dev/null @@ -1,187 +0,0 @@ -package grpc_test - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "testing" - - pb "github.com/Open-MBEE/OpenSysML/api/proto" - "github.com/Open-MBEE/OpenSysML/internal/frontend/grpc" - "github.com/Open-MBEE/OpenSysML/internal/translate/convert" - "github.com/Open-MBEE/OpenSysML/tests/testutil/corpussubset" -) - -// agreementExamples are the committed single-document models under examples/; -// the downloaded corpora have gates of their own. -func agreementExamples(t *testing.T) []string { - t.Helper() - root := filepath.Join("..", "..", "examples") - subset := corpussubset.Must(t, "../..") - var paths []string - err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - if entry.IsDir() && entry.Name() == "pilot-corpora" { - return filepath.SkipDir - } - if ext := filepath.Ext(path); !entry.IsDir() && (ext == ".sysml" || ext == ".kerml") && subset.WantsPath(root, path) { - paths = append(paths, path) - } - return nil - }) - if err != nil { - t.Fatal(err) - } - sort.Strings(paths) - return paths -} - -// GetElementProperties answers what the api-json export writes, for every -// (element, metamodel property) pair of every example: the same names, the same -// values in the same order, element values by the same id; the export's `sysx:` -// extension keys are no metamodel properties. A hygiene check, not a baseline. -func TestElementPropertiesAgreeWithTheAPIJSONExport(t *testing.T) { - paths := agreementExamples(t) - if len(paths) == 0 { - t.Fatal("no example models") - } - srv, err := grpc.NewService(4, "test") - if err != nil { - t.Fatal(err) - } - ctx := context.Background() - converted, elements, pairs := 0, 0, 0 - for _, path := range paths { - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - exported, err := convert.Convert(filepath.Base(path), data, convert.FormatSysML, convert.FormatAPIJSON) - if err != nil { - continue - } - absolute, err := filepath.Abs(path) - if err != nil { - t.Fatal(err) - } - parsed, err := srv.ParseFile(ctx, &pb.ParseFileRequest{Source: &pb.ParseFileRequest_FilePath{FilePath: absolute}}) - if err != nil { - t.Fatalf("%s: ParseFile: %v", path, err) - } - var objects []map[string]any - decoder := json.NewDecoder(strings.NewReader(string(exported))) - decoder.UseNumber() - if err := decoder.Decode(&objects); err != nil { - t.Fatalf("%s: api-json: %v", path, err) - } - converted++ - for _, object := range objects { - id, _ := object["@id"].(string) - resp, err := srv.GetElementProperties(ctx, &pb.GetElementPropertiesRequest{ModelHash: parsed.ModelHash, ElementId: id}) - if err != nil { - t.Fatalf("%s: GetElementProperties(%s): %v", path, id, err) - } - if resp.Error != "" { - t.Errorf("%s: GetElementProperties(%s): %s", path, id, resp.Error) - continue - } - if metaclass, _ := object["@type"].(string); resp.Element.Metaclass != metaclass { - t.Errorf("%s: %s is a %s in the export and a %s in the answer", path, id, metaclass, resp.Element.Metaclass) - } - elements++ - answered := map[string]*pb.ElementProperty{} - for _, property := range resp.Properties { - answered[property.Name] = property - } - for name, value := range object { - if name == "@id" || name == "@type" || strings.HasPrefix(name, "sysx:") { - continue - } - pairs++ - property, ok := answered[name] - if !ok { - t.Errorf("%s: %s.%s is exported but not answered", path, id, name) - continue - } - delete(answered, name) - if want, got := apiJSONSpelling(value), propertySpelling(property); want != got { - t.Errorf("%s: %s.%s exported as %s, answered as %s", path, id, name, want, got) - } - } - for name := range answered { - t.Errorf("%s: %s.%s is answered but not exported", path, id, name) - } - } - } - if converted == 0 || elements == 0 { - t.Fatalf("no example converts to api-json (%d models, %d elements)", converted, elements) - } - t.Logf("%d of %d examples convert; %d elements, %d (element, property) pairs agree", converted, len(paths), elements, pairs) -} - -// apiJSONSpelling flattens an exported property value to one comparable text: -// element references by id, literals by value, in collection order. -func apiJSONSpelling(value any) string { - switch v := value.(type) { - case nil: - return "" - case []any: - parts := make([]string, 0, len(v)) - for _, member := range v { - parts = append(parts, apiJSONSpelling(member)) - } - return strings.Join(parts, " | ") - case map[string]any: - id, _ := v["@id"].(string) - return "element " + id - case json.Number: - return numberSpelling(v.String()) - case string: - return "string " + v - case bool: - return "bool " + strconv.FormatBool(v) - } - return fmt.Sprintf("%T %v", value, value) -} - -// propertySpelling flattens an answered property the way apiJSONSpelling does the export. -func propertySpelling(property *pb.ElementProperty) string { - if property.Unsupported != "" { - return "unsupported: " + property.Unsupported - } - parts := make([]string, 0, len(property.Values)) - for _, value := range property.Values { - switch v := value.Value.(type) { - case *pb.ElementPropertyValue_Element: - parts = append(parts, "element "+v.Element.Id) - case *pb.ElementPropertyValue_StringValue: - parts = append(parts, "string "+v.StringValue) - case *pb.ElementPropertyValue_BoolValue: - parts = append(parts, "bool "+strconv.FormatBool(v.BoolValue)) - case *pb.ElementPropertyValue_IntValue: - parts = append(parts, numberSpelling(strconv.FormatInt(v.IntValue, 10))) - case *pb.ElementPropertyValue_RealValue: - parts = append(parts, numberSpelling(strconv.FormatFloat(v.RealValue, 'g', -1, 64))) - case *pb.ElementPropertyValue_Number: - parts = append(parts, numberSpelling(v.Number)) - default: - parts = append(parts, fmt.Sprintf("%T", value.Value)) - } - } - return strings.Join(parts, " | ") -} - -// numberSpelling normalizes a decimal text so 2.0 and 2 compare equal. -func numberSpelling(text string) string { - if f, err := strconv.ParseFloat(text, 64); err == nil { - return "number " + strconv.FormatFloat(f, 'g', -1, 64) - } - return "number " + text -} diff --git a/tests/grpc/element_properties_census_test.go b/tests/grpc/element_properties_census_test.go index 4180654671..d9ef7e20cd 100644 --- a/tests/grpc/element_properties_census_test.go +++ b/tests/grpc/element_properties_census_test.go @@ -17,6 +17,31 @@ import ( "github.com/Open-MBEE/OpenSysML/internal/translate/rdf/ontology" ) +// agreementExamples are the committed single-document models under examples/; +// the downloaded corpora have gates of their own. +func agreementExamples(t *testing.T) []string { + t.Helper() + root := filepath.Join("..", "..", "examples") + var paths []string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && entry.Name() == "pilot-corpora" { + return filepath.SkipDir + } + if ext := filepath.Ext(path); !entry.IsDir() && (ext == ".sysml" || ext == ".kerml") { + paths = append(paths, path) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + sort.Strings(paths) + return paths +} + // censusWitness is one element of one model whose metaclass carries a // metamodel property under the defining class a rule or omission names. type censusWitness struct { diff --git a/tests/hygiene/race_shards_test.go b/tests/hygiene/race_shards_test.go index cb87dc461c..7f9b4d5402 100644 --- a/tests/hygiene/race_shards_test.go +++ b/tests/hygiene/race_shards_test.go @@ -13,10 +13,8 @@ import ( "testing" ) -// The race shards are named in five places that must agree: scripts/race-shard.sh, -// the Makefile (test-shard, RACE_RUNTIME_CORPUS, RACE_EXPORT_CORPUS, RACE_EXPORT_SETTLED, -// RACE_CORPUS_ROUNDTRIP, RACE_CORPUS_APIJSON, COVERAGE_SHARDS), -// the two CircleCI jobs' shard arrays and parallelism, and the PR workflow's matrices. +// The race shards are named in five places that must agree: scripts/race-shard.sh, the Makefile +// (test-shard, RACE_*, COVERAGE_SHARDS), the CircleCI shard arrays and the PR workflow's matrices. const ( raceRuntimeDir = "../../internal/exec/runtime" @@ -30,16 +28,12 @@ var ( makefileCorpus = regexp.MustCompile(`(?m)^RACE_RUNTIME_CORPUS := (.+)$`) makefileExport = regexp.MustCompile(`(?m)^RACE_EXPORT_CORPUS := (.+)$`) circleExport = regexp.MustCompile(`(?m)^\s*exportcorpus='([^']+)'$`) - makefileSettled = regexp.MustCompile(`(?m)^RACE_EXPORT_SETTLED := (.+)$`) - circleSettled = regexp.MustCompile(`(?m)^\s*exportsettled='([^']+)'$`) makefileLibrary = regexp.MustCompile(`(?m)^RACE_EXPORT_LIBRARY := (.+)$`) circleLibrary = regexp.MustCompile(`(?m)^\s*exportlibrary='([^']+)'$`) makefileExamples = regexp.MustCompile(`(?m)^RACE_EXPORT_EXAMPLES := (.+)$`) circleExamples = regexp.MustCompile(`(?m)^\s*exportexamples='([^']+)'$`) - makefileRoundTrip = regexp.MustCompile(`(?m)^RACE_CORPUS_ROUNDTRIP := (.+)$`) - circleRoundTrip = regexp.MustCompile(`(?m)^\s*corpusroundtrip='([^']+)'$`) - makefileAPIJSON = regexp.MustCompile(`(?m)^RACE_CORPUS_APIJSON := (.+)$`) - circleAPIJSON = regexp.MustCompile(`(?m)^\s*corpusapijson='([^']+)'$`) + makefileSweep = regexp.MustCompile(`(?m)^RACE_CORPUS_SWEEP := (.+)$`) + circleSweep = regexp.MustCompile(`(?m)^\s*corpussweep='([^']+)'$`) makefileCoverage = regexp.MustCompile(`(?m)^COVERAGE_SHARDS := (.+)$`) circleShardArrays = regexp.MustCompile(`(?m)^\s*shards=\(([^)]+)\)$`) shardArrayEntry = regexp.MustCompile(`"([^"]*)"|(\S+)`) @@ -149,7 +143,7 @@ func shardArrayEntries(body string) []string { // test-quick runs a group's shards in turn, with their selections, without -race). // isCorpusShard says whether a shard is a by-test-name part of another shard's package -// (runtime-corpus, export-corpus, export-settled, corpus-roundtrip, ...): the hyphenated names. +// (runtime-corpus, export-corpus, corpus-sweep, ...): the hyphenated names. func isCorpusShard(shard string) bool { return strings.Contains(shard, "-") } func TestPRWorkflowMatricesCoverTheRaceShards(t *testing.T) { shards := raceShards(t) @@ -206,16 +200,15 @@ func TestRuntimeCorpusPatternPartitionsTheRuntimeTests(t *testing.T) { } } -// TestExportCorpusPatternsPartitionTheExportTests: the five export shards are tests/export split the -// same way by the four RACE_EXPORT_* patterns, of which no test may match more than one. +// TestExportCorpusPatternsPartitionTheExportTests: the four export shards are tests/export split the +// same way by the three RACE_EXPORT_* patterns, of which no test may match more than one. func TestExportCorpusPatternsPartitionTheExportTests(t *testing.T) { corpus, tests := corpusPatternPartitions(t, "RACE_EXPORT_CORPUS", makefileExport, circleExport, "exportcorpus=", raceExportDir, 50) - settled, _ := corpusPatternPartitions(t, "RACE_EXPORT_SETTLED", makefileSettled, circleSettled, "exportsettled=", raceExportDir, 50) library, _ := corpusPatternPartitions(t, "RACE_EXPORT_LIBRARY", makefileLibrary, circleLibrary, "exportlibrary=", raceExportDir, 50) examples, _ := corpusPatternPartitions(t, "RACE_EXPORT_EXAMPLES", makefileExamples, circleExamples, "exportexamples=", raceExportDir, 50) for _, name := range tests { matches := 0 - for _, pattern := range []*regexp.Regexp{corpus, settled, library, examples} { + for _, pattern := range []*regexp.Regexp{corpus, library, examples} { if pattern.MatchString(name) { matches++ } @@ -226,17 +219,10 @@ func TestExportCorpusPatternsPartitionTheExportTests(t *testing.T) { } } -// TestCorpusPatternsPartitionTheCorpusTests: the corpus, corpus-roundtrip and corpus-apijson -// shards are tests/corpus split the same way by RACE_CORPUS_ROUNDTRIP and RACE_CORPUS_APIJSON, -// which no test may match both of. +// TestCorpusPatternsPartitionTheCorpusTests: the corpus and corpus-sweep shards are +// tests/corpus split the same way by RACE_CORPUS_SWEEP. func TestCorpusPatternsPartitionTheCorpusTests(t *testing.T) { - roundTrip, tests := corpusPatternPartitions(t, "RACE_CORPUS_ROUNDTRIP", makefileRoundTrip, circleRoundTrip, "corpusroundtrip=", raceCorpusDir, 10) - apiJSON, _ := corpusPatternPartitions(t, "RACE_CORPUS_APIJSON", makefileAPIJSON, circleAPIJSON, "corpusapijson=", raceCorpusDir, 10) - for _, name := range tests { - if roundTrip.MatchString(name) && apiJSON.MatchString(name) { - t.Errorf("%s falls to both corpus-roundtrip and corpus-apijson", name) - } - } + corpusPatternPartitions(t, "RACE_CORPUS_SWEEP", makefileSweep, circleSweep, "corpussweep=", raceCorpusDir, 10) } // corpusPatternPartitions checks one by-test-name split: the Makefile variable and the