From 903a1b22d54dd663862ef9f76b4e0bfd6f70617c Mon Sep 17 00:00:00 2001 From: Antigravity Agent Date: Wed, 11 Nov 2026 01:28:45 -0600 Subject: [PATCH] feat(git): add configurable commit history retention and get_recent_changes MCP tool (closes #101) --- REQUIREMENTS.md | 13 +- app/api/routers/repositories.py | 36 ++- app/mcp/handlers/__init__.py | 2 + app/mcp/handlers/repo_handlers.py | 118 +++++++++- app/mcp/tools.py | 7 + app/models/schemas.py | 4 + app/services/database/connection.py | 20 +- app/services/database/schema.py | 17 ++ app/services/git_manager.py | 108 ++++++++- app/services/indexing/git_syncer.py | 54 ++++- frontend/src/GitRepoManager.tsx | 29 +++ frontend/src/components/git/AddRepoModal.tsx | 24 +- frontend/src/components/git/RepoListTable.tsx | 62 ++++- frontend/src/tests/GitRepoManager.test.tsx | 3 +- .../src/tests/RetainCommitsControls.test.tsx | 84 +++++++ frontend/src/types.ts | 1 + tests/backend/test_api_routes.py | 3 +- tests/backend/test_git_commits_history.py | 215 ++++++++++++++++++ 18 files changed, 775 insertions(+), 25 deletions(-) create mode 100644 frontend/src/tests/RetainCommitsControls.test.tsx create mode 100644 tests/backend/test_git_commits_history.py diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index fbe7de8..0d6f3ea 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1151 Automated Tests** (793 Pytest Backend + 308 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1158 Automated Tests** (798 Pytest Backend + 310 Vitest Frontend + 50 Playwright E2E). --- @@ -515,6 +515,13 @@ classDiagram - `test_ring_buffer_emit_exception_handling` - `test_logs_api_routes` +#### `tests/backend/test_git_commits_history.py` (5 tests) +- `test_extract_recent_commits_unit` +- `test_format_git_commit_url` +- `test_sync_single_git_repo_commits_and_pruning` +- `test_get_recent_changes_mcp_tool` +- `test_rest_api_patch_retain_commits` + #### `tests/backend/test_git_manager.py` (34 tests) - `TestGitManager::test_get_env_token` - `TestGitManager::test_normalize_git_url` @@ -1584,6 +1591,10 @@ over the PostgresConnectionAdapter using an initialized test engine._ - cancels sync by calling /admin/api/repos/{id}/cancel-sync - closes EventSource on unmount +#### `RetainCommitsControls.test.tsx` (2 tests) +- renders retain commits input on AddRepoModal and calls setRetainCommitsCount +- renders history retention badge on RepoListTable and prompts for edit + #### `SearchInspector.test.tsx` (11 tests) - renders initial prompt and inputs - switches search mode and updates UI controls diff --git a/app/api/routers/repositories.py b/app/api/routers/repositories.py index 01c5370..682d3c7 100644 --- a/app/api/routers/repositories.py +++ b/app/api/routers/repositories.py @@ -11,7 +11,7 @@ from app.services.auth import Role from app.api.routers.auth import require_role -from app.models.schemas import RepoConfig, LocalPathConfig, AutoSyncToggleRequest, KeepShallowToggleRequest, SearchRequest +from app.models.schemas import RepoConfig, LocalPathConfig, AutoSyncToggleRequest, KeepShallowToggleRequest, RetainCommitsRequest, SearchRequest import app.services.database as db_service import app.services.vector_store as vs_service import app.services.indexing as idx_service @@ -32,7 +32,7 @@ async def api_get_repos(): with db_service.get_db_connection() as conn: rows = conn.execute(""" SELECT id, name, url, branch, commit_sha, status, last_error, last_synced, - auth_token, provider, auth_user, auto_sync, keep_shallow, webhook_secret + auth_token, provider, auth_user, auto_sync, keep_shallow, retain_commits_count, webhook_secret FROM git_repositories ORDER BY id DESC """).fetchall() @@ -61,11 +61,12 @@ async def api_add_repo(repo: RepoConfig): try: with db_service.get_db_connection() as conn: cursor = conn.cursor() + retain_count = repo.retain_commits_count if repo.retain_commits_count is not None else 20 cursor.execute( - """INSERT INTO git_repositories (name, url, branch, auth_token, provider, auth_user, auto_sync, keep_shallow, webhook_secret) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + """INSERT INTO git_repositories (name, url, branch, auth_token, provider, auth_user, auto_sync, keep_shallow, retain_commits_count, webhook_secret) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (repo.name.strip(), repo.url.strip(), repo.branch or "main", repo.auth_token, repo.provider or "github", repo.auth_user, - 1 if repo.auto_sync else 0, 1 if repo.keep_shallow else 0, repo.webhook_secret) + 1 if repo.auto_sync else 0, 1 if repo.keep_shallow else 0, max(0, retain_count), repo.webhook_secret) ) conn.commit() repo_id = cursor.lastrowid @@ -114,6 +115,29 @@ async def api_toggle_repo_keep_shallow(repo_id: int, payload: KeepShallowToggleR logger.error(f"Error toggling keep-shallow for repo {repo_id}: {e}") return JSONResponse(status_code=500, content={"error": "Failed to toggle keep-shallow for repository."}) +@router.patch( + "/admin/api/repos/{repo_id}/retain-commits", + dependencies=[Depends(require_role(Role.ADMIN))], +) +async def api_update_repo_retain_commits(repo_id: int, payload: RetainCommitsRequest): + try: + with db_service.get_db_connection() as conn: + cursor = conn.cursor() + val = max(0, payload.retain_commits_count) + res = cursor.execute("UPDATE git_repositories SET retain_commits_count = ? WHERE id = ?", (val, repo_id)) + if res.rowcount == 0: + return JSONResponse(status_code=404, content={"error": f"Repository with ID {repo_id} not found"}) + if val == 0: + # Delete any stored commits immediately if retention is disabled + row = cursor.execute("SELECT name FROM git_repositories WHERE id = ?", (repo_id,)).fetchone() + if row: + cursor.execute("DELETE FROM git_commits WHERE repo = ?", (row["name"],)) + conn.commit() + return {"status": "success", "id": repo_id, "repo_id": repo_id, "retain_commits_count": val} + except Exception as e: + logger.error(f"Error updating retain_commits_count for repo {repo_id}: {e}") + return JSONResponse(status_code=500, content={"error": "Failed to update retain_commits_count for repository."}) + @router.post( "/admin/api/repos/{repo_id}/sync", dependencies=[Depends(require_role(Role.EDITOR))], @@ -218,7 +242,7 @@ async def api_delete_repo(repo_id: int): name = row["name"] conn.execute("DELETE FROM git_repositories WHERE id = ?", (repo_id,)) - for table in ("indexed_files", "ast_symbols", "ast_relationships", "api_routes"): + for table in ("indexed_files", "ast_symbols", "ast_relationships", "api_routes", "git_commits"): try: conn.execute(f"DELETE FROM {table} WHERE repo = ?", (name,)) except Exception: diff --git a/app/mcp/handlers/__init__.py b/app/mcp/handlers/__init__.py index 90fa55d..742e685 100644 --- a/app/mcp/handlers/__init__.py +++ b/app/mcp/handlers/__init__.py @@ -18,6 +18,7 @@ handle_index_status, handle_catalog_summary, handle_search_infrastructure_docs, + handle_get_recent_changes, ) from app.mcp.handlers.architecture_handlers import ( handle_get_architecture, @@ -46,6 +47,7 @@ "handle_index_status", "handle_catalog_summary", "handle_search_infrastructure_docs", + "handle_get_recent_changes", "handle_get_architecture", "handle_manage_adr", "handle_manage_local_file", diff --git a/app/mcp/handlers/repo_handlers.py b/app/mcp/handlers/repo_handlers.py index e0924e9..12c1b82 100644 --- a/app/mcp/handlers/repo_handlers.py +++ b/app/mcp/handlers/repo_handlers.py @@ -7,7 +7,7 @@ from app.services.search import execute_hybrid_search, trace_symbol_path from app.services.indexing import get_dynamic_catalog_description, sync_single_git_repo, run_full_indexing from app.services.vector_store import get_vector_store_config -from app.services.git_manager import format_git_permalink +from app.services.git_manager import format_git_permalink, format_git_commit_url from app.services.chunking import match_route_and_call, normalize_path_pattern from app.models.schemas import SearchRequest, FindSymbolRequest, GetFileOutlineRequest, SyncRequest @@ -175,3 +175,119 @@ def handle_search_infrastructure_docs( return f"Please perform a search using the search_docs tool for topic '{topic}' and summarize the matching container mappings, port numbers, reverse proxy routes, or setup instructions." +async def handle_get_recent_changes( + repo: Annotated[str, Field(description="Repository name to inspect recent commits for.")], + filepath: Annotated[Optional[str], Field(description="Optional relative file path filter to only return commits touching this file.")] = None, + symbol: Annotated[Optional[str], Field(description="Optional AST symbol name; resolves symbol's file path and filters commits touching that file.")] = None, + limit: Annotated[int, Field(description="Max commits to return (default 10, max 50).")] = 10, +) -> str: + """Returns recent commit history for a repository, optionally filtered by file path or symbol name.""" + try: + from app.services.auth import enforce_tool_permission, Role + enforce_tool_permission(Role.VIEWER) + + safe_limit = max(1, min(limit if limit is not None else 10, 50)) + + with _get_tools_attr("get_db_connection", get_db_connection)() as conn: + repo_row = conn.execute( + "SELECT id, name, url, provider, retain_commits_count FROM git_repositories WHERE name = ?", + (repo,) + ).fetchone() + + retained_limit = ( + int(repo_row["retain_commits_count"]) + if repo_row and "retain_commits_count" in repo_row.keys() and repo_row["retain_commits_count"] is not None + else 20 + ) + + # Check if repository has commits in git_commits + stored_count = 0 + if repo_row: + count_row = conn.execute("SELECT count(*) as c FROM git_commits WHERE repo = ?", (repo,)).fetchone() + stored_count = count_row["c"] if count_row else 0 + + # If retain_commits_count == 0 or no commits are stored + if not repo_row or retained_limit == 0 or stored_count == 0: + fallback_resp = { + "repo": repo, + "retained": False, + "retained_limit": retained_limit if repo_row else 0, + "commits": [], + "message": f"Commit history is not retained for repository '{repo}' (retain_commits_count={retained_limit if repo_row else 0}). Retrieve commit history directly from the Git repository." + } + return json.dumps(fallback_resp, separators=(",", ":")) + + git_url = repo_row["url"] + provider = repo_row["provider"] + + # Resolve filter filepath + target_filepath = None + if filepath and filepath.strip(): + clean_fp = filepath.strip().lstrip("/") + target_filepath = clean_fp + elif symbol and symbol.strip(): + sym_row = conn.execute( + "SELECT filepath FROM ast_symbols WHERE repo = ? AND (name = ? OR full_symbol = ?) LIMIT 1", + (repo, symbol.strip(), symbol.strip()) + ).fetchone() + if sym_row: + raw_fp = sym_row["filepath"] + if "://" in raw_fp: + target_filepath = raw_fp.split("://", 1)[1] + else: + target_filepath = raw_fp + + # Query commits from git_commits + rows = conn.execute( + "SELECT commit_sha, author, committed_at, message, files_changed FROM git_commits WHERE repo = ? ORDER BY committed_at DESC, id DESC", + (repo,) + ).fetchall() + + matched_commits = [] + for r in rows: + files_raw = r["files_changed"] + try: + files_list = json.loads(files_raw) if files_raw else [] + except Exception: + files_list = [] + + if target_filepath: + norm_target = target_filepath.replace("\\", "/").lstrip("/") + matched = False + for f in files_list: + norm_f = f.replace("\\", "/").lstrip("/") + if norm_f == norm_target or norm_f.endswith("/" + norm_target) or norm_target.endswith("/" + norm_f): + matched = True + break + if not matched: + continue + + commit_url = format_git_commit_url(git_url, r["commit_sha"], provider=provider) + matched_commits.append({ + "sha": r["commit_sha"], + "short_sha": r["commit_sha"][:7], + "author": r["author"], + "date": r["committed_at"], + "message": r["message"], + "files_changed": files_list, + "commit_url": commit_url, + }) + + if len(matched_commits) >= safe_limit: + break + + result = { + "repo": repo, + "retained_limit": retained_limit, + "commits": matched_commits, + } + if target_filepath: + result["filter_filepath"] = target_filepath + + return json.dumps(result, separators=(",", ":")) + + except Exception as e: + logger.error(f"Error in handle_get_recent_changes: {e}") + return json.dumps({"error": str(e), "repo": repo, "commits": []}, separators=(",", ":")) + + diff --git a/app/mcp/tools.py b/app/mcp/tools.py index dedf98b..62ed417 100644 --- a/app/mcp/tools.py +++ b/app/mcp/tools.py @@ -24,6 +24,7 @@ handle_index_status, handle_catalog_summary, handle_search_infrastructure_docs, + handle_get_recent_changes, handle_get_architecture, handle_manage_adr, handle_manage_local_file, @@ -90,6 +91,12 @@ def register_mcp_tools_and_resources(server=None): description="Lists all indexed local paths and remote Git repositories, including active branches, commit SHAs, and file counts." )(handle_list_repositories) + if "get_recent_changes" not in existing_tools: + server.tool( + name="get_recent_changes", + description="Inspect recent commits for a Git repository, optionally filtered by file path or symbol name." + )(handle_get_recent_changes) + if "sync_repository" not in existing_tools: server.tool( name="sync_repository", diff --git a/app/models/schemas.py b/app/models/schemas.py index eaf54db..9bb53ff 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -17,6 +17,7 @@ class RepoConfig(BaseModel): enabled: bool = True auto_sync: bool = True keep_shallow: bool = False + retain_commits_count: int = 20 webhook_secret: Optional[str] = None class LocalPathConfig(BaseModel): @@ -172,6 +173,9 @@ class AutoSyncToggleRequest(BaseModel): class KeepShallowToggleRequest(BaseModel): keep_shallow: bool +class RetainCommitsRequest(BaseModel): + retain_commits_count: int = Field(default=20, ge=0, le=1000) + class AutoSyncSettingsRequest(BaseModel): interval_mins: int global_webhook_secret: Optional[str] = None diff --git a/app/services/database/connection.py b/app/services/database/connection.py index 6c43e06..a223718 100644 --- a/app/services/database/connection.py +++ b/app/services/database/connection.py @@ -446,7 +446,7 @@ def set_ai_gateway_config( return get_ai_gateway_config() def ensure_git_repositories_columns(conn=None): - """Ensures keep_shallow column exists on git_repositories table.""" + """Ensures keep_shallow and retain_commits_count columns exist on git_repositories table.""" try: if is_postgres(): from sqlalchemy import text @@ -458,6 +458,14 @@ def ensure_git_repositories_columns(conn=None): sa_conn.execute(text("ALTER TABLE git_repositories ADD COLUMN keep_shallow INTEGER DEFAULT 0")) sa_conn.commit() logger.info("Migrated git_repositories: added keep_shallow column") + + res_retain = sa_conn.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name='git_repositories' AND column_name='retain_commits_count'") + ).fetchone() + if not res_retain: + sa_conn.execute(text("ALTER TABLE git_repositories ADD COLUMN retain_commits_count INTEGER DEFAULT 20")) + sa_conn.commit() + logger.info("Migrated git_repositories: added retain_commits_count column") else: raw_conn = get_db_connection() try: @@ -466,13 +474,17 @@ def ensure_git_repositories_columns(conn=None): raw_conn.execute("ALTER TABLE git_repositories ADD COLUMN keep_shallow INTEGER DEFAULT 0") raw_conn.commit() logger.info("Migrated git_repositories: added keep_shallow column") + if cols and "retain_commits_count" not in cols: + raw_conn.execute("ALTER TABLE git_repositories ADD COLUMN retain_commits_count INTEGER DEFAULT 20") + raw_conn.commit() + logger.info("Migrated git_repositories: added retain_commits_count column") finally: raw_conn.close() except Exception as e: logger.debug(f"Migration error for git_repositories: {e}") def ensure_file_summaries_columns(conn=None): - """Ensures summary_text column exists on file_summaries table and git_repositories has keep_shallow.""" + """Ensures summary_text column exists on file_summaries table and git_repositories has keep_shallow and retain_commits_count.""" try: ensure_git_repositories_columns(conn) if is_postgres(): @@ -568,12 +580,12 @@ def set_file_settings(payload: Dict[str, Any]) -> Dict[str, Any]: def clear_indexing_metadata(): """ Clears all relational index tables (indexed_files, file_summaries, ast_symbols, - ast_relationships, api_routes, api_client_calls, embedding_cache) and resets + ast_relationships, api_routes, api_client_calls, embedding_cache, git_commits) and resets git repository sync markers so that subsequent reindexing passes fully re-process and re-embed all sources. """ with get_db_connection() as conn: - for table in ("indexed_files", "file_summaries", "ast_symbols", "ast_relationships", "api_routes", "api_client_calls", "embedding_cache"): + for table in ("indexed_files", "file_summaries", "ast_symbols", "ast_relationships", "api_routes", "api_client_calls", "embedding_cache", "git_commits"): try: conn.execute(f"DELETE FROM {table}") except Exception as e: diff --git a/app/services/database/schema.py b/app/services/database/schema.py index db3089b..f8b7167 100644 --- a/app/services/database/schema.py +++ b/app/services/database/schema.py @@ -50,6 +50,7 @@ Column("enabled", Integer, default=1, server_default="1"), Column("auto_sync", Integer, default=1, server_default="1"), Column("keep_shallow", Integer, default=0, server_default="0"), + Column("retain_commits_count", Integer, default=20, server_default="20"), Column("webhook_secret", Text, nullable=True), Column("status", Text, default="pending", server_default="pending"), Column("last_error", Text, nullable=True), @@ -255,6 +256,21 @@ Index("idx_pdf_cache_repo", "repo"), ) +# 16. Git Commits +git_commits = Table( + "git_commits", + metadata, + Column("id", Integer, primary_key=True, autoincrement=True), + Column("repo", Text, nullable=False), + Column("commit_sha", Text, nullable=False), + Column("author", Text, nullable=True), + Column("committed_at", Text, nullable=True), + Column("message", Text, nullable=True), + Column("files_changed", Text, nullable=True), # JSON array of relative filepaths + Index("idx_git_commits_repo_sha", "repo", "commit_sha", unique=True), + Index("idx_git_commits_repo_date", "repo", "committed_at"), +) + # Dictionary mapping table names to SQLAlchemy Table objects TABLES: Dict[str, Table] = {t.name: t for t in metadata.tables.values()} @@ -263,6 +279,7 @@ "TABLES", "indexed_paths", "git_repositories", + "git_commits", "git_host_credentials", "indexed_files", "file_summaries", diff --git a/app/services/git_manager.py b/app/services/git_manager.py index 7991104..c86c1e0 100644 --- a/app/services/git_manager.py +++ b/app/services/git_manager.py @@ -201,7 +201,8 @@ def shallow_clone_repo( username: Optional[str] = None, provider: Optional[str] = None, repo_id: Optional[str] = None, - target_dir: Optional[str] = None + target_dir: Optional[str] = None, + depth: int = 1 ) -> CloneResult: """ Shallow clones a git repository to a temporary or persistent directory. @@ -211,6 +212,7 @@ def shallow_clone_repo( from app.services.database import get_effective_git_token norm_url = normalize_git_url(git_url) use_ssh_native = is_ssh_git_url(git_url) and not token and not username + depth_str = str(max(1, depth)) if target_dir: os.makedirs(os.path.dirname(os.path.abspath(target_dir)), exist_ok=True) @@ -228,12 +230,12 @@ def shallow_clone_repo( auth_url = build_authenticated_url(target_url, token, username=username, provider=provider) safe_url = sanitize_url_for_logging(auth_url) - logger.info(f"Cloning {safe_url} (branch: {branch}) shallowly to {repo_dir}...") + logger.info(f"Cloning {safe_url} (branch: {branch}, depth: {depth_str}) shallowly to {repo_dir}...") try: # Check if already a git repo in target_dir (incremental shallow update) if target_dir and os.path.exists(os.path.join(repo_dir, ".git")): - logger.info(f"Existing shallow clone found in {repo_dir}; fetching latest refs...") - fetch_cmd = ["git", "fetch", "--depth", "1", "origin", branch] + logger.info(f"Existing shallow clone found in {repo_dir}; fetching latest refs (depth: {depth_str})...") + fetch_cmd = ["git", "fetch", "--depth", depth_str, "origin", branch] f_res = subprocess.run(fetch_cmd, cwd=repo_dir, capture_output=True, text=True, timeout=60, env=GIT_ENV) if f_res.returncode == 0: subprocess.run(["git", "reset", "--hard", f"origin/{branch}"], cwd=repo_dir, capture_output=True, text=True, timeout=30, env=GIT_ENV) @@ -250,7 +252,7 @@ def shallow_clone_repo( # Shallow clone single branch cmd = [ "git", "clone", - "--depth", "1", + "--depth", depth_str, "--branch", branch, "--single-branch", auth_url, @@ -259,7 +261,7 @@ def shallow_clone_repo( res = subprocess.run(cmd, capture_output=True, text=True, timeout=120, env=GIT_ENV) if res.returncode != 0: # Fallback to default clone if branch failed (e.g. branch is default) - cmd_fallback = ["git", "clone", "--depth", "1", auth_url, repo_dir] + cmd_fallback = ["git", "clone", "--depth", depth_str, auth_url, repo_dir] res = subprocess.run(cmd_fallback, capture_output=True, text=True, timeout=120, env=GIT_ENV) if res.returncode != 0: err_msg = res.stderr.strip() or res.stdout.strip() or "Git clone failed" @@ -277,6 +279,77 @@ def shallow_clone_repo( cleanup_repo_dir(repo_dir) return CloneResult(temp_dir=None, commit_sha=None, error=f"Exception during git clone: {str(e)}") +def extract_recent_commits(repo_dir: str, limit: int = 20) -> list[Dict[str, Any]]: + """ + Extracts recent commit metadata from a local git repository. + Returns: List of dicts with commit_sha, author, committed_at (ISO 8601), + message (commit subject + body), and files_changed (list of relative filepaths). + """ + if not repo_dir or not os.path.exists(os.path.join(repo_dir, ".git")): + return [] + if limit <= 0: + return [] + + # Delimiters for structured git log parsing + record_delim = "===CC_COMMIT_START===" + files_delim = "===CC_FILES_START===" + + # Format: record_delim, SHA, author name , ISO date (%cI), raw body (%B), files_delim + cmd = [ + "git", "log", + f"-n{limit}", + "--name-only", + f"--pretty=format:{record_delim}%n%H%n%an <%ae>%n%cI%n%B{files_delim}", + ] + + try: + res = subprocess.run(cmd, cwd=repo_dir, capture_output=True, text=True, timeout=30, env=GIT_ENV) + if res.returncode != 0 or not res.stdout: + return [] + + raw_output = res.stdout + chunks = raw_output.split(record_delim) + commits = [] + + for chunk in chunks: + chunk = chunk.strip() + if not chunk: + continue + + if files_delim not in chunk: + continue + + meta_part, files_part = chunk.split(files_delim, 1) + meta_lines = meta_part.splitlines() + if len(meta_lines) < 3: + continue + + commit_sha = meta_lines[0].strip() + author = meta_lines[1].strip() + committed_at = meta_lines[2].strip() + # Everything after the first 3 lines is the commit message + message = "\n".join(meta_lines[3:]).strip() if len(meta_lines) > 3 else "" + + # files_part contains the relative filepaths touched by the commit + files_changed = [] + for f_line in files_part.splitlines(): + f_path = f_line.strip() + if f_path and f_path not in files_changed: + files_changed.append(f_path) + + commits.append({ + "commit_sha": commit_sha, + "author": author, + "committed_at": committed_at, + "message": message, + "files_changed": files_changed, + }) + + return commits + except Exception as e: + logger.error(f"Error extracting recent commits from {repo_dir}: {e}") + return [] + def cleanup_repo_dir(repo_dir: Optional[str]): """Safely removes cloned temporary directory from disk.""" if repo_dir and os.path.exists(repo_dir): @@ -286,6 +359,29 @@ def cleanup_repo_dir(repo_dir: Optional[str]): except Exception as e: logger.warning(f"Failed to cleanup temp directory {repo_dir}: {e}") +def format_git_commit_url( + git_url: str, + commit_sha: str, + provider: Optional[str] = None +) -> Optional[str]: + """Constructs a clickable web URL for a commit (e.g. GitHub, GitLab, Gitea, Bitbucket).""" + if not git_url or not commit_sha: + return None + base = normalize_git_url(git_url).rstrip("/") + if base.endswith(".git"): + base = base[:-4] + + prov = detect_git_provider(base, provider) + if prov == "gitlab": + return f"{base}/-/commit/{commit_sha}" + elif prov == "bitbucket": + return f"{base}/commits/{commit_sha}" + elif prov == "gitea": + return f"{base}/commit/{commit_sha}" + else: + # GitHub and generic standard + return f"{base}/commit/{commit_sha}" + def format_git_permalink( git_url: str, commit_sha: Optional[str], diff --git a/app/services/indexing/git_syncer.py b/app/services/indexing/git_syncer.py index a3fe7cd..69c6945 100644 --- a/app/services/indexing/git_syncer.py +++ b/app/services/indexing/git_syncer.py @@ -111,6 +111,7 @@ def sync_single_git_repo(repo_id: int): per_repo_user = repo_row["auth_user"] if "auth_user" in repo_row.keys() else None provider = repo_row["provider"] if "provider" in repo_row.keys() else None keep_shallow = bool(repo_row["keep_shallow"]) if "keep_shallow" in repo_row.keys() and repo_row["keep_shallow"] else False + retain_commits_count = int(repo_row["retain_commits_count"]) if "retain_commits_count" in repo_row.keys() and repo_row["retain_commits_count"] is not None else 20 progress_tracker.get_or_create_job(repo_id, repo_name) @@ -149,8 +150,9 @@ def sync_single_git_repo(repo_id: int): target_dir = gm_service.get_persistent_repo_dir(repo_name) if keep_shallow else None clone_mode_label = f"persistent storage ({target_dir})" if keep_shallow else "ephemeral directory" + clone_depth = max(1, retain_commits_count) if retain_commits_count > 0 else 1 progress_tracker.update_step(repo_id, 2, "Shallow Cloning Repository", pct=25) - progress_tracker.log(repo_id, "INFO", f"Cloning branch '{branch}' to {clone_mode_label}...") + progress_tracker.log(repo_id, "INFO", f"Cloning branch '{branch}' (depth {clone_depth}) to {clone_mode_label}...") clone_start = time.time() clone_res = gm_service.shallow_clone_repo( @@ -160,7 +162,8 @@ def sync_single_git_repo(repo_id: int): username=effective_user, provider=provider, repo_id=str(repo_id), - target_dir=target_dir + target_dir=target_dir, + depth=clone_depth ) temp_dir = clone_res.temp_dir commit_sha = clone_res.commit_sha @@ -184,6 +187,53 @@ def sync_single_git_repo(repo_id: int): progress_tracker.log(repo_id, "WARN", "Sync cancelled by user") return + # Commit history extraction and pruning + try: + with db_service.get_db_connection() as conn: + if retain_commits_count > 0: + recent_commits = gm_service.extract_recent_commits(temp_dir, limit=retain_commits_count) + if recent_commits: + commit_tuples = [ + ( + repo_name, + c["commit_sha"], + c.get("author"), + c.get("committed_at"), + c.get("message"), + json.dumps(c.get("files_changed", [])) + ) + for c in recent_commits + ] + conn.executemany( + """INSERT INTO git_commits (repo, commit_sha, author, committed_at, message, files_changed) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(repo, commit_sha) DO UPDATE SET + author = EXCLUDED.author, + committed_at = EXCLUDED.committed_at, + message = EXCLUDED.message, + files_changed = EXCLUDED.files_changed""", + commit_tuples + ) + # Prune older rows beyond retain_commits_count for this repo + # We keep the top retain_commits_count commits ordered by committed_at DESC, id DESC + conn.execute( + """DELETE FROM git_commits + WHERE repo = ? AND id NOT IN ( + SELECT id FROM git_commits + WHERE repo = ? + ORDER BY committed_at DESC, id DESC + LIMIT ? + )""", + (repo_name, repo_name, retain_commits_count) + ) + conn.commit() + else: + # retain_commits_count <= 0: delete all stored commits for this repo + conn.execute("DELETE FROM git_commits WHERE repo = ?", (repo_name,)) + conn.commit() + except Exception as ce: + logger.error(f"Error extracting/pruning commits for repo '{repo_name}': {ce}") + # Stage 3: Computing File Delta & Scanning progress_tracker.update_step(repo_id, 3, "Computing File Delta & Scanning", pct=40) added_files, modified_files, deleted_filepaths, unchanged_files = compute_git_repo_delta(temp_dir, repo_name) diff --git a/frontend/src/GitRepoManager.tsx b/frontend/src/GitRepoManager.tsx index 6479a0e..f8445a6 100644 --- a/frontend/src/GitRepoManager.tsx +++ b/frontend/src/GitRepoManager.tsx @@ -34,6 +34,7 @@ export default function GitRepoManager({ refreshStats }: { refreshStats: () => v const [authUser, setAuthUser] = useState(''); const [token, setToken] = useState(''); const [keepShallow, setKeepShallow] = useState(false); + const [retainCommitsCount, setRetainCommitsCount] = useState(20); const [isSaving, setIsSaving] = useState(false); const loadRepos = useCallback(async () => { @@ -74,6 +75,7 @@ export default function GitRepoManager({ refreshStats }: { refreshStats: () => v auth_user: authUser.trim() || null, auth_token: token.trim() || null, keep_shallow: keepShallow, + retain_commits_count: retainCommitsCount, }), }); const data = await res.json(); @@ -87,6 +89,7 @@ export default function GitRepoManager({ refreshStats }: { refreshStats: () => v setAuthUser(''); setToken(''); setKeepShallow(false); + setRetainCommitsCount(20); loadRepos(); refreshStats(); toast.success(`Repository '${alias.trim()}' added successfully`); @@ -162,6 +165,29 @@ export default function GitRepoManager({ refreshStats }: { refreshStats: () => v } }; + const updateRetainCommits = async (repoId: number, count: number) => { + setRepos((prev) => + prev.map((r) => (r.id === repoId ? { ...r, retain_commits_count: count } : r)) + ); + + try { + const res = await fetch(`/admin/api/repos/${repoId}/retain-commits`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ retain_commits_count: count }), + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.error || data.detail || 'Failed to update commit history retention'); + } + toast.info(`Commit history retention updated to ${count} commits`); + loadRepos(); + } catch (e: any) { + loadRepos(); + toast.error('Failed to update commit history retention: ' + e.message); + } + }; + const handleCopyUrl = async (text: string) => { try { if (navigator?.clipboard?.writeText) { @@ -230,6 +256,7 @@ export default function GitRepoManager({ refreshStats }: { refreshStats: () => v onSync={syncRepo} onToggleAutoSync={toggleAutoSync} onToggleKeepShallow={toggleKeepShallow} + onUpdateRetainCommits={updateRetainCommits} onOpenWebhook={(repo) => setWebhookModalRepo(repo)} onDelete={deleteRepo} onOpenSyncDrawer={(id) => setActiveDrawerRepoId(id)} @@ -254,6 +281,8 @@ export default function GitRepoManager({ refreshStats }: { refreshStats: () => v setToken={setToken} keepShallow={keepShallow} setKeepShallow={setKeepShallow} + retainCommitsCount={retainCommitsCount} + setRetainCommitsCount={setRetainCommitsCount} isSaving={isSaving} /> diff --git a/frontend/src/components/git/AddRepoModal.tsx b/frontend/src/components/git/AddRepoModal.tsx index e7c592b..ec86724 100644 --- a/frontend/src/components/git/AddRepoModal.tsx +++ b/frontend/src/components/git/AddRepoModal.tsx @@ -19,6 +19,8 @@ interface AddRepoModalProps { setToken: (val: string) => void; keepShallow?: boolean; setKeepShallow?: (val: boolean) => void; + retainCommitsCount?: number; + setRetainCommitsCount?: (val: number) => void; isSaving: boolean; } @@ -40,6 +42,8 @@ export function AddRepoModal({ setToken, keepShallow = false, setKeepShallow, + retainCommitsCount = 20, + setRetainCommitsCount, isSaving, }: AddRepoModalProps) { if (!isOpen) return null; @@ -148,7 +152,25 @@ export function AddRepoModal({ Keep shallow copy on disk

- When enabled, retains a shallow clone (--depth 1) in persistent storage on disk after indexing, enabling instant full-file inspection in Code Navigator and faster delta updates. + When enabled, retains a shallow clone in persistent storage on disk after indexing, enabling instant full-file inspection in Code Navigator and faster delta updates. +

+ + +
+ + setRetainCommitsCount?.(Math.max(0, parseInt(e.target.value, 10) || 0))} + style={{ width: '100%', maxWidth: '160px', marginTop: '4px' }} + /> +

+ Number of recent commits to index and retain for get_recent_changes queries (default 20, set to 0 to disable commit history retention).

diff --git a/frontend/src/components/git/RepoListTable.tsx b/frontend/src/components/git/RepoListTable.tsx index 7918401..cfc0921 100644 --- a/frontend/src/components/git/RepoListTable.tsx +++ b/frontend/src/components/git/RepoListTable.tsx @@ -7,6 +7,7 @@ interface RepoListTableProps { onSync: (id: number) => void; onToggleAutoSync: (id: number, current: boolean) => void; onToggleKeepShallow?: (id: number, current: boolean) => void; + onUpdateRetainCommits?: (id: number, count: number) => void; onOpenWebhook: (repo: Repo) => void; onDelete: (id: number, name: string) => void; onOpenSyncDrawer?: (repoId: number) => void; @@ -19,6 +20,7 @@ export function RepoListTable({ onSync, onToggleAutoSync, onToggleKeepShallow, + onUpdateRetainCommits, onOpenWebhook, onDelete, onOpenSyncDrawer, @@ -87,6 +89,7 @@ export function RepoListTable({ Commit SHA Status Auto-Sync + History Retained Shallow Copy Files Last Synced @@ -96,14 +99,14 @@ export function RepoListTable({ {isLoading && repos.length === 0 ? ( - + Loading repositories... ) : repos.length === 0 ? ( - + No Git repositories registered. Click "Add Repository" to index a remote repo. @@ -111,6 +114,7 @@ export function RepoListTable({ repos.map((r) => { const isAutoSync = r.auto_sync !== false && r.auto_sync !== 0; const isKeepShallow = r.keep_shallow === true || r.keep_shallow === 1; + const retainCount = r.retain_commits_count ?? 20; const job = syncStates?.[r.id]; const isSyncing = r.status === 'syncing' || job?.status === 'syncing'; @@ -177,6 +181,38 @@ export function RepoListTable({ )} + + + +
Shallow Copy: