Skip to content

Commit 45d6283

Browse files
ericyangpanericyangpan
committed
feat(data): add manifest maintenance harness
1 parent c2218fe commit 45d6283

26 files changed

Lines changed: 1519 additions & 347 deletions

‎.github/workflows/monitor-model-sources.yml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,19 @@ jobs:
2424
- name: Setup pnpm
2525
uses: ./.github/actions/setup-pnpm
2626

27-
- name: Check monitored official sources
28-
run: pnpm fetch:model-sources
27+
- name: Update and validate monitored official sources
28+
run: pnpm manifest-data:update -- --only=model-sources
2929

30-
- name: Validate monitoring metadata
30+
- name: Check formatting
3131
run: |
32-
pnpm test:validate
3332
pnpm format:check
3433
pnpm biome:check
3534
git diff --check
3635
3736
- name: Check for changes
3837
id: git-check
3938
run: |
40-
git diff --exit-code manifests/models || echo "changes=true" >> "$GITHUB_OUTPUT"
39+
git diff --exit-code manifests/models data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
4140
4241
- name: Create review pull request
4342
if: steps.git-check.outputs.changes == 'true'

‎.github/workflows/update-benchmarks.yml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,19 @@ jobs:
2424
- name: Setup pnpm
2525
uses: ./.github/actions/setup-pnpm
2626

27-
- name: Refresh exact benchmark mappings
28-
run: pnpm fetch:benchmarks
27+
- name: Update and validate exact benchmark mappings
28+
run: pnpm manifest-data:update -- --only=benchmarks
2929

30-
- name: Regenerate manifest modules
31-
run: pnpm generate:manifests
32-
33-
- name: Validate updated data
30+
- name: Check formatting
3431
run: |
35-
pnpm test:validate
36-
pnpm data-health:check
3732
pnpm format:check
3833
pnpm biome:check
3934
git diff --check
4035
4136
- name: Check for changes
4237
id: git-check
4338
run: |
44-
git diff --exit-code manifests src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
39+
git diff --exit-code manifests data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
4540
4641
- name: Create pull request
4742
if: steps.git-check.outputs.changes == 'true'

‎.github/workflows/update-github-stars.yml‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,15 @@ jobs:
2424
- name: Setup pnpm
2525
uses: ./.github/actions/setup-pnpm
2626

27-
- name: Fetch GitHub stars
28-
run: pnpm fetch:github-stars
27+
- name: Update and validate GitHub stars
28+
run: pnpm manifest-data:update -- --only=github-stars
2929
env:
3030
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3131

32-
- name: Generate TypeScript files
33-
run: pnpm generate:manifests
34-
3532
- name: Check for changes
3633
id: git-check
3734
run: |
38-
git diff --exit-code data/github-stars.json src/lib/generated/github-stars.ts || echo "changes=true" >> $GITHUB_OUTPUT
35+
git diff --exit-code data/github-stars.json data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
3936
4037
- name: Create Pull Request
4138
if: steps.git-check.outputs.changes == 'true'
@@ -50,7 +47,7 @@ jobs:
5047
5148
### Changes
5249
- Updated `data/github-stars.json` with latest star counts
53-
- Regenerated `src/lib/generated/github-stars.ts`
50+
- Regenerated derived data and the data-health snapshot
5451
5552
### Source
5653
This PR was automatically generated by the `update-github-stars` workflow.

‎.github/workflows/update-product-versions.yml‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,21 @@ jobs:
2424
- name: Setup pnpm
2525
uses: ./.github/actions/setup-pnpm
2626

27-
- name: Refresh tracked product versions
28-
run: pnpm fetch:product-versions
27+
- name: Update and validate tracked product versions
28+
run: pnpm manifest-data:update -- --only=product-versions
2929
env:
3030
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3131

32-
- name: Regenerate manifest modules
33-
run: pnpm generate:manifests
34-
35-
- name: Validate updated data
32+
- name: Check formatting
3633
run: |
37-
pnpm test:validate
38-
pnpm validate:i18n
39-
pnpm data-health:check
4034
pnpm format:check
4135
pnpm biome:check
4236
git diff --check
4337
4438
- name: Check for changes
4539
id: git-check
4640
run: |
47-
git diff --exit-code manifests src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
41+
git diff --exit-code manifests data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
4842
4943
- name: Create pull request
5044
if: steps.git-check.outputs.changes == 'true'

‎data/$schemas/data-health.schema.json‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"title": "Manifest Data Health Report",
4+
"description": "Generated manifest provenance, freshness, relationship, and translation health snapshot.",
5+
"type": "object",
6+
"properties": {
7+
"asOf": { "type": "string", "format": "date" },
8+
"thresholds": { "$ref": "#/$defs/categoryThresholds" },
9+
"summary": { "$ref": "#/$defs/summary" },
10+
"byCategory": { "$ref": "#/$defs/categoryHealthMap" },
11+
"translationsByLocale": {
12+
"type": "object",
13+
"minProperties": 1,
14+
"additionalProperties": { "$ref": "#/$defs/translationHealth" }
15+
},
16+
"issues": {
17+
"type": "array",
18+
"items": { "$ref": "#/$defs/issue" }
19+
}
20+
},
21+
"required": ["asOf", "thresholds", "summary", "byCategory", "translationsByLocale", "issues"],
22+
"additionalProperties": false,
23+
"$defs": {
24+
"categories": {
25+
"type": "string",
26+
"enum": ["ides", "clis", "desktops", "extensions", "models", "providers", "vendors"]
27+
},
28+
"nonNegativeInteger": {
29+
"type": "integer",
30+
"minimum": 0
31+
},
32+
"percentage": {
33+
"type": "number",
34+
"minimum": 0,
35+
"maximum": 100
36+
},
37+
"categoryThresholds": {
38+
"type": "object",
39+
"properties": {
40+
"ides": { "$ref": "#/$defs/nonNegativeInteger" },
41+
"clis": { "$ref": "#/$defs/nonNegativeInteger" },
42+
"desktops": { "$ref": "#/$defs/nonNegativeInteger" },
43+
"extensions": { "$ref": "#/$defs/nonNegativeInteger" },
44+
"models": { "$ref": "#/$defs/nonNegativeInteger" },
45+
"providers": { "$ref": "#/$defs/nonNegativeInteger" },
46+
"vendors": { "$ref": "#/$defs/nonNegativeInteger" }
47+
},
48+
"required": ["ides", "clis", "desktops", "extensions", "models", "providers", "vendors"],
49+
"additionalProperties": false
50+
},
51+
"summary": {
52+
"type": "object",
53+
"properties": {
54+
"totalRecords": { "$ref": "#/$defs/nonNegativeInteger" },
55+
"recordsWithSources": { "$ref": "#/$defs/nonNegativeInteger" },
56+
"verifiedRecords": { "$ref": "#/$defs/nonNegativeInteger" },
57+
"provenanceComplete": { "$ref": "#/$defs/nonNegativeInteger" },
58+
"staleVerifiedRecords": { "$ref": "#/$defs/nonNegativeInteger" },
59+
"translationPlaceholderValues": { "$ref": "#/$defs/nonNegativeInteger" },
60+
"danglingRelationships": { "$ref": "#/$defs/nonNegativeInteger" },
61+
"modelBenchmarkCoverage": { "$ref": "#/$defs/percentage" },
62+
"productsWithPricing": { "$ref": "#/$defs/nonNegativeInteger" },
63+
"productRecords": { "$ref": "#/$defs/nonNegativeInteger" },
64+
"communityUrlsPopulated": { "$ref": "#/$defs/nonNegativeInteger" },
65+
"communityUrlsWithProvenance": { "$ref": "#/$defs/nonNegativeInteger" },
66+
"duplicatedVendorCommunityUrls": { "$ref": "#/$defs/nonNegativeInteger" },
67+
"errors": { "$ref": "#/$defs/nonNegativeInteger" },
68+
"warnings": { "$ref": "#/$defs/nonNegativeInteger" },
69+
"info": { "$ref": "#/$defs/nonNegativeInteger" }
70+
},
71+
"required": [
72+
"totalRecords",
73+
"recordsWithSources",
74+
"verifiedRecords",
75+
"provenanceComplete",
76+
"staleVerifiedRecords",
77+
"translationPlaceholderValues",
78+
"danglingRelationships",
79+
"modelBenchmarkCoverage",
80+
"productsWithPricing",
81+
"productRecords",
82+
"communityUrlsPopulated",
83+
"communityUrlsWithProvenance",
84+
"duplicatedVendorCommunityUrls",
85+
"errors",
86+
"warnings",
87+
"info"
88+
],
89+
"additionalProperties": false
90+
},
91+
"categoryHealth": {
92+
"type": "object",
93+
"properties": {
94+
"total": { "$ref": "#/$defs/nonNegativeInteger" },
95+
"verified": { "$ref": "#/$defs/nonNegativeInteger" },
96+
"provenanceComplete": { "$ref": "#/$defs/nonNegativeInteger" },
97+
"stale": { "$ref": "#/$defs/nonNegativeInteger" }
98+
},
99+
"required": ["total", "verified", "provenanceComplete", "stale"],
100+
"additionalProperties": false
101+
},
102+
"categoryHealthMap": {
103+
"type": "object",
104+
"properties": {
105+
"ides": { "$ref": "#/$defs/categoryHealth" },
106+
"clis": { "$ref": "#/$defs/categoryHealth" },
107+
"desktops": { "$ref": "#/$defs/categoryHealth" },
108+
"extensions": { "$ref": "#/$defs/categoryHealth" },
109+
"models": { "$ref": "#/$defs/categoryHealth" },
110+
"providers": { "$ref": "#/$defs/categoryHealth" },
111+
"vendors": { "$ref": "#/$defs/categoryHealth" }
112+
},
113+
"required": ["ides", "clis", "desktops", "extensions", "models", "providers", "vendors"],
114+
"additionalProperties": false
115+
},
116+
"translationHealth": {
117+
"type": "object",
118+
"properties": {
119+
"totalStrings": { "$ref": "#/$defs/nonNegativeInteger" },
120+
"matchingEnglish": { "$ref": "#/$defs/nonNegativeInteger" },
121+
"matchingEnglishPercent": { "$ref": "#/$defs/percentage" }
122+
},
123+
"required": ["totalStrings", "matchingEnglish", "matchingEnglishPercent"],
124+
"additionalProperties": false
125+
},
126+
"issue": {
127+
"type": "object",
128+
"properties": {
129+
"severity": { "type": "string", "enum": ["error", "warning", "info"] },
130+
"code": { "type": "string", "minLength": 1 },
131+
"category": { "$ref": "#/$defs/categories" },
132+
"id": { "type": "string", "minLength": 1 },
133+
"message": { "type": "string", "minLength": 1 }
134+
},
135+
"required": ["severity", "code", "category", "id", "message"],
136+
"additionalProperties": false
137+
}
138+
}
139+
}

‎docs/DATA-TRUST.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ Every new model catalog entry must have a published Artificial Analysis Intellig
5252

5353
## Freshness policy
5454

55-
Run `pnpm data-health:report` after manifest or translation changes and commit both generated snapshots. CI runs `pnpm data-health:check` to reject invalid or stale snapshots.
55+
`pnpm manifest-data:validate` is the offline validation entry point for schema coverage, semantic relationships, i18n structure, and the committed data-health snapshot. `pnpm manifest-data:check` adds read-only checks against every configured authoritative source. `pnpm manifest-data:update` updates only the explicitly configured safe fields, regenerates derived data and health reports, and reruns the offline validation suite. Source checks can be limited with `--only=<source,...>`.
56+
57+
The update harness covers GitHub star snapshots, declared product release versions, exact benchmark mappings, and monitored model-source digests. A pricing or lifecycle source change updates only its digest and observation date; changing the corresponding factual fields still requires review against the cited official source. Any missing, malformed, renamed, or ambiguous upstream result fails the relevant task.
58+
59+
Run `pnpm data-health:report` after manifest or translation changes made outside the unified update harness and commit both generated snapshots. CI runs `pnpm data-health:check` to reject invalid or stale snapshots.
5660

5761
The current review thresholds are 30 days for models and providers, 60 days for IDEs, CLIs, and extensions, and 90 days for vendors. Threshold findings remain warnings so that legacy debt stays visible without blocking unrelated work. Dangling product relationships are errors because they always point to nonexistent records and can break navigation. Network reachability is checked by the separate scheduled URL workflow; source authority and field coverage remain pull-request review responsibilities.

‎manifests/$schemas/github-stars.schema.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"$schema": "http://json-schema.org/draft-07/schema#",
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
33
"$id": "https://aicodingstack.io/schemas/github-stars.schema.json",
44
"title": "GitHub Repository Stars Data",
55
"description": "Repository-keyed GitHub star snapshots. Product associations and source-code status are derived from product manifests.",

‎manifests/$schemas/mapping.schema.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"title": "External Manifest ID Mapping",
4+
"description": "Explicit vendor and model identifier mappings used by external benchmark comparisons.",
5+
"type": "object",
6+
"properties": {
7+
"vendors": { "$ref": "#/$defs/mapping" },
8+
"models": { "$ref": "#/$defs/mapping" }
9+
},
10+
"required": ["vendors", "models"],
11+
"additionalProperties": false,
12+
"$defs": {
13+
"mapping": {
14+
"type": "object",
15+
"propertyNames": { "minLength": 1 },
16+
"additionalProperties": {
17+
"type": "string",
18+
"minLength": 1
19+
}
20+
}
21+
}
22+
}

‎manifests/$schemas/ref/product.schema.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@
149149
"identifier": {
150150
"type": "string",
151151
"pattern": "^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$"
152+
},
153+
"tagPrefix": {
154+
"type": "string",
155+
"minLength": 1,
156+
"description": "Exact GitHub release tag prefix removed before updating latestVersion"
152157
}
153158
},
154159
"required": ["provider", "identifier"],

‎manifests/desktops/reasonix-desktop.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@
8181
"latestVersion": "1.34.0",
8282
"releaseTracking": {
8383
"provider": "github-release",
84-
"identifier": "esengine/DeepSeek-Reasonix"
84+
"identifier": "esengine/DeepSeek-Reasonix",
85+
"tagPrefix": "desktop-"
8586
},
8687
"githubUrl": "https://github.com/esengine/DeepSeek-Reasonix",
8788
"license": "MIT",

0 commit comments

Comments
 (0)