Skip to content

Show newly discovered collections on the Dashboard #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ewlarson merged 1 commit into from
Oct 6, 2026
Merged

Show newly discovered collections on the Dashboard #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions backend/app/api/v1/endpoint_modules/ogm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,15 @@ async def ogm_repo_dashboard(request: Request):
catalog = {r["ogm_repo_name"]: r for r in await ogm_repo.list_public_repo_summaries()}
dashboard_repos = []
enabled_repos = 0
for name, count in sorted(index_status["repo_counts"].items()):
if count <= 0:
continue
indexed_counts = index_status["repo_counts"]
visible_names = {name for name, count in indexed_counts.items() if count > 0}
visible_names.update(
name
for name, repo in catalog.items()
if repo.get("ogm_enabled") and not repo.get("ogm_archived", False)
)
for name in sorted(visible_names):
count = indexed_counts.get(name, 0)
repo = catalog.get(name, {"ogm_repo_name": name})
enabled = bool(repo.get("ogm_enabled")) and not repo.get("ogm_archived", False)
scheduled = enabled and repo.get("ogm_watch_mode") in {
Expand Down Expand Up @@ -89,6 +95,15 @@ async def ogm_repo_dashboard(request: Request):
"thumbnail_refresh": "Refreshing thumbnails",
"dumps": "Saving harvest files",
}.get(repo.get("last_run_stage"), "Processing"),
"display_harvest_status": repo.get("last_crawl_status")
or (
"Awaiting first harvest"
if enabled
and not repo.get("last_crawl_started_at")
and not repo.get("last_successful_harvest_at")
and count == 0
else "Unknown"
),
"source_status": "Nightly" if scheduled else "Not scheduled",
"indexed_record_count": count,
}
Expand All @@ -112,7 +127,7 @@ async def ogm_repo_dashboard(request: Request):
"<td>"
f"{escape(repo.get('display_last_successful_harvest_at') or no_success)}"
"</td>"
f"<td>{escape(str(repo.get('last_crawl_status') or 'Unknown'))}"
f"<td>{escape(str(repo['display_harvest_status']))}"
+ (
f" — {escape(repo['display_harvest_stage'])}"
if repo.get("last_crawl_status") == "running"
Expand Down
12 changes: 6 additions & 6 deletions backend/templates/ogm_repo_dashboard.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,17 @@ <h1><a class="navbar-brand" href="/api/v1/ogm/repos/dashboard" data-tagline="Rep
<main class="docs-wrap">
<section class="monitor-hero">
<p class="eyebrow">OpenGeoMetadata Harvest Monitor</p>
<h2>Repositories in the search index.</h2>
<h2>Repository discovery and harvest progress.</h2>
<p class="lede">
Repositories with records in the live search index. Each nightly sync fetches the latest
Enabled collections appear here as soon as they are discovered, including those awaiting their first harvest. Each nightly sync fetches the latest
metadata, imports it, and updates the index. Use Search API to browse a repository’s records.
</p>
<p class="meta-note">Generated {{ generated_at }}. Times shown in UTC.</p>
</section>

<section class="monitor-summary" aria-label="Repository summary">
<article class="summary-card">
<p class="summary-label">Indexed Repositories</p>
<p class="summary-label">Monitored Repositories</p>
<p class="summary-value">{{ summary.repo_count }}</p>
</article>
<article class="summary-card">
Expand Down Expand Up @@ -108,7 +108,7 @@ <h3>Repositories</h3>
{% elif crawl_status == "failed" %}
<span class="status-pill status-bad">Failed</span>
{% else %}
<span class="status-pill status-muted">{{ repo.last_crawl_status or "Unknown" }}</span>
<span class="status-pill status-muted">{{ repo.display_harvest_status }}</span>
{% endif %}
{% if repo.display_harvest_started_at %}
<p class="repo-subtitle">Started {{ repo.display_harvest_started_at }}</p>
Expand All @@ -120,7 +120,7 @@ <h3>Repositories</h3>
<td>{{ "{:,}".format(repo.indexed_record_count) }}</td>
</tr>
{% else %}
<tr><td colspan="5">No repositories currently have indexed records.</td></tr>
<tr><td colspan="5">No enabled or indexed repositories found.</td></tr>
{% endfor %}
</tbody>
</table>
Expand All @@ -146,7 +146,7 @@ <h3 class="mt-3">Monitor Links</h3>
<div class="col-xs-12 col-lg-6">
<h3 class="mt-3">Notes</h3>
<ul class="list-unstyled">
<li>Only repositories represented in the search index appear above.</li>
<li>Enabled repositories appear before their first harvest; repositories with indexed records remain visible.</li>
<li>The searchable total includes records without a repository tag.</li>
<li>Nightly runs refresh metadata and synchronize each repository’s search records.</li>
</ul>
Expand Down
44 changes: 41 additions & 3 deletions backend/tests/api/v1/test_ogm_public_endpoints.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ def test_public_ogm_failures_endpoint_can_limit_to_hard_failures_only():
)


def test_dashboard_only_shows_indexed_sources_even_when_database_disagrees():
def test_dashboard_shows_enabled_sources_but_keeps_counts_from_index():
repos = [
{"ogm_repo_name": "retired", "ogm_enabled": False, "available_record_count": 100},
{"ogm_repo_name": "empty", "ogm_enabled": True, "available_record_count": 10},
Expand All @@ -198,7 +198,15 @@ def test_dashboard_only_shows_indexed_sources_even_when_database_disagrees():
):
response = client.get("/api/v1/ogm/repos/dashboard")
assert response.status_code == 200
for expected in ("318,408", "40,703", "geobtaa", "uncataloged", "Scheduled Nightly"):
for expected in (
"318,408",
"40,703",
"geobtaa",
"uncataloged",
"empty",
"Awaiting first harvest",
"Scheduled Nightly",
):
assert expected in response.text
for absent in ("retired", "Never Harvested Here", "Database", "also in active sources"):
assert absent not in response.text
Expand All @@ -221,7 +229,7 @@ def test_dashboard_empty_index():
):
response = client.get("/api/v1/ogm/repos/dashboard")
assert response.status_code == 200
assert "No repositories currently have indexed records" in response.text
assert "No enabled or indexed repositories found" in response.text


@pytest.mark.parametrize(
Expand Down Expand Up @@ -282,3 +290,33 @@ def test_dashboard_does_not_invent_success_from_latest_completion(status):
assert "No successful harvest recorded" in html
assert "2026-10-04 14:00 UTC" not in html
assert ("Updating search index" in html) == (status == "running")


@pytest.mark.parametrize("fallback", [False, True])
def test_new_collection_visible_before_indexing(fallback):
repos = [
{
"ogm_repo_name": "gov.usgs.tnm",
"ogm_enabled": True,
"ogm_watch_mode": "both",
"ogm_has_aardvark": True,
},
{"ogm_repo_name": "archived-source", "ogm_enabled": True, "ogm_archived": True},
{"ogm_repo_name": "disabled-source", "ogm_enabled": False},
{"ogm_repo_name": "failed-source", "ogm_enabled": True, "last_crawl_status": "failed"},
]
with (
patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=repos)),
patch.object(
ogm, "get_index_status", AsyncMock(return_value={"record_count": 0, "repo_counts": {}})
),
patch.object(ogm, "templates", None if fallback else ogm.templates),
):
response = client.get("/api/v1/ogm/repos/dashboard")
assert response.status_code == 200
assert "gov.usgs.tnm" in response.text
assert "Awaiting first harvest" in response.text
assert "failed-source" in response.text
assert "failed" in response.text.lower()
assert "archived-source" not in response.text
assert "disabled-source" not in response.text
Loading