From 241e99cd1bb2dd97a1a175c029f503ca054b6676 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 6 Oct 2026 09:45:00 -0500 Subject: [PATCH] Show enabled collections before their first indexed record --- backend/app/api/v1/endpoint_modules/ogm.py | 23 ++++++++-- backend/templates/ogm_repo_dashboard.html | 12 ++--- .../tests/api/v1/test_ogm_public_endpoints.py | 44 +++++++++++++++++-- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/backend/app/api/v1/endpoint_modules/ogm.py b/backend/app/api/v1/endpoint_modules/ogm.py index 9b986d6..f5724d6 100644 --- a/backend/app/api/v1/endpoint_modules/ogm.py +++ b/backend/app/api/v1/endpoint_modules/ogm.py @@ -59,9 +59,15 @@ async def ogm_repo_dashboard(request: Request): catalog = {r["ogm_repo_name"]: r for r in await ogm_repo.list_public_repo_summaries()} dashboard_repos = [] enabled_repos = 0 - for name, count in sorted(index_status["repo_counts"].items()): - if count <= 0: - continue + indexed_counts = index_status["repo_counts"] + visible_names = {name for name, count in indexed_counts.items() if count > 0} + visible_names.update( + name + for name, repo in catalog.items() + if repo.get("ogm_enabled") and not repo.get("ogm_archived", False) + ) + for name in sorted(visible_names): + count = indexed_counts.get(name, 0) repo = catalog.get(name, {"ogm_repo_name": name}) enabled = bool(repo.get("ogm_enabled")) and not repo.get("ogm_archived", False) scheduled = enabled and repo.get("ogm_watch_mode") in { @@ -89,6 +95,15 @@ async def ogm_repo_dashboard(request: Request): "thumbnail_refresh": "Refreshing thumbnails", "dumps": "Saving harvest files", }.get(repo.get("last_run_stage"), "Processing"), + "display_harvest_status": repo.get("last_crawl_status") + or ( + "Awaiting first harvest" + if enabled + and not repo.get("last_crawl_started_at") + and not repo.get("last_successful_harvest_at") + and count == 0 + else "Unknown" + ), "source_status": "Nightly" if scheduled else "Not scheduled", "indexed_record_count": count, } @@ -112,7 +127,7 @@ async def ogm_repo_dashboard(request: Request): "" f"{escape(repo.get('display_last_successful_harvest_at') or no_success)}" "" - f"{escape(str(repo.get('last_crawl_status') or 'Unknown'))}" + f"{escape(str(repo['display_harvest_status']))}" + ( f" — {escape(repo['display_harvest_stage'])}" if repo.get("last_crawl_status") == "running" diff --git a/backend/templates/ogm_repo_dashboard.html b/backend/templates/ogm_repo_dashboard.html index 532712d..1a85f50 100644 --- a/backend/templates/ogm_repo_dashboard.html +++ b/backend/templates/ogm_repo_dashboard.html @@ -33,9 +33,9 @@

OpenGeoMetadata Harvest Monitor

-

Repositories in the search index.

+

Repository discovery and harvest progress.

- Repositories with records in the live search index. Each nightly sync fetches the latest + Enabled collections appear here as soon as they are discovered, including those awaiting their first harvest. Each nightly sync fetches the latest metadata, imports it, and updates the index. Use Search API to browse a repository’s records.

Generated {{ generated_at }}. Times shown in UTC.

@@ -43,7 +43,7 @@

Repositories in the search index.

-

Indexed Repositories

+

Monitored Repositories

{{ summary.repo_count }}

@@ -108,7 +108,7 @@

Repositories

{% elif crawl_status == "failed" %} Failed {% else %} - {{ repo.last_crawl_status or "Unknown" }} + {{ repo.display_harvest_status }} {% endif %} {% if repo.display_harvest_started_at %}

Started {{ repo.display_harvest_started_at }}

@@ -120,7 +120,7 @@

Repositories

{{ "{:,}".format(repo.indexed_record_count) }} {% else %} - No repositories currently have indexed records. + No enabled or indexed repositories found. {% endfor %} @@ -146,7 +146,7 @@

Monitor Links

Notes

    -
  • Only repositories represented in the search index appear above.
    • +
  • Enabled repositories appear before their first harvest; repositories with indexed records remain visible.
  • The searchable total includes records without a repository tag.
  • Nightly runs refresh metadata and synchronize each repository’s search records.
diff --git a/backend/tests/api/v1/test_ogm_public_endpoints.py b/backend/tests/api/v1/test_ogm_public_endpoints.py index 02ed6d2..3715f53 100644 --- a/backend/tests/api/v1/test_ogm_public_endpoints.py +++ b/backend/tests/api/v1/test_ogm_public_endpoints.py @@ -172,7 +172,7 @@ def test_public_ogm_failures_endpoint_can_limit_to_hard_failures_only(): ) -def test_dashboard_only_shows_indexed_sources_even_when_database_disagrees(): +def test_dashboard_shows_enabled_sources_but_keeps_counts_from_index(): repos = [ {"ogm_repo_name": "retired", "ogm_enabled": False, "available_record_count": 100}, {"ogm_repo_name": "empty", "ogm_enabled": True, "available_record_count": 10}, @@ -198,7 +198,15 @@ def test_dashboard_only_shows_indexed_sources_even_when_database_disagrees(): ): response = client.get("/api/v1/ogm/repos/dashboard") assert response.status_code == 200 - for expected in ("318,408", "40,703", "geobtaa", "uncataloged", "Scheduled Nightly"): + for expected in ( + "318,408", + "40,703", + "geobtaa", + "uncataloged", + "empty", + "Awaiting first harvest", + "Scheduled Nightly", + ): assert expected in response.text for absent in ("retired", "Never Harvested Here", "Database", "also in active sources"): assert absent not in response.text @@ -221,7 +229,7 @@ def test_dashboard_empty_index(): ): response = client.get("/api/v1/ogm/repos/dashboard") assert response.status_code == 200 - assert "No repositories currently have indexed records" in response.text + assert "No enabled or indexed repositories found" in response.text @pytest.mark.parametrize( @@ -282,3 +290,33 @@ def test_dashboard_does_not_invent_success_from_latest_completion(status): assert "No successful harvest recorded" in html assert "2026-10-04 14:00 UTC" not in html assert ("Updating search index" in html) == (status == "running") + + +@pytest.mark.parametrize("fallback", [False, True]) +def test_new_collection_visible_before_indexing(fallback): + repos = [ + { + "ogm_repo_name": "gov.usgs.tnm", + "ogm_enabled": True, + "ogm_watch_mode": "both", + "ogm_has_aardvark": True, + }, + {"ogm_repo_name": "archived-source", "ogm_enabled": True, "ogm_archived": True}, + {"ogm_repo_name": "disabled-source", "ogm_enabled": False}, + {"ogm_repo_name": "failed-source", "ogm_enabled": True, "last_crawl_status": "failed"}, + ] + with ( + patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=repos)), + patch.object( + ogm, "get_index_status", AsyncMock(return_value={"record_count": 0, "repo_counts": {}}) + ), + patch.object(ogm, "templates", None if fallback else ogm.templates), + ): + response = client.get("/api/v1/ogm/repos/dashboard") + assert response.status_code == 200 + assert "gov.usgs.tnm" in response.text + assert "Awaiting first harvest" in response.text + assert "failed-source" in response.text + assert "failed" in response.text.lower() + assert "archived-source" not in response.text + assert "disabled-source" not in response.text