refactor(gitextractor): replace libgit2 and go-git with the git CLI

.devcontainer/Dockerfile

@@ -15,10 +15,7 @@
 
 FROM mcr.microsoft.com/devcontainers/go:2-1.27-bookworm
 
-ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib:/usr/local/lib/pkgconfig
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 ENV DEVLAKE_PLUGINS=argocd,bamboo,bitbucket,circleci,customize,dora,gitextractor,github,github_graphql,gitlab,issue_trace,jenkins,jira,org,pagerduty,refdiff,slack,sonarqube,trello,webhook
 
 RUN apt-get update -y
-RUN apt-get install pkg-config python3-dev default-libmysqlclient-dev build-essential libpq-dev cmake -y
-RUN git clone -b v1.5.0 https://github.com/libgit2/libgit2.git && cd libgit2 && mkdir build && cd build && cmake .. && make && make install
+RUN apt-get install python3-dev default-libmysqlclient-dev build-essential libpq-dev -y

.github/dependabot.yml

@@ -58,9 +58,6 @@ updates:
       # The Go toolchain directive is coupled to the CI bootstrap version and
       # to the golang base images; it is raised deliberately, never by a bot.
       - dependency-name: go
-      # git2go is ABI-coupled to the libgit2 version installed in the builder
-      # image. A mismatch fails at link time, so this is always manual.
-      - dependency-name: github.com/libgit2/git2go/*
       # swag v2 is still a release candidate and requires regenerating every
       # swagger annotation. Tracked as an externally gated upgrade.
       - dependency-name: github.com/swaggo/swag

.github/workflows/golangci-lint.yml

@@ -32,8 +32,6 @@ jobs:
       - uses: actions/checkout@v7
       - name: Set git config
         run: git config --global --add safe.directory $(pwd)
-      - name: Install libgit2 1.5 for git2go v34
-        run: backend/scripts/install-libgit2.sh
       - name: Install Go 1.27.3
         run: backend/scripts/install-go.sh
       - name: Install mockery 3.8.1

.github/workflows/test-e2e.yml

@@ -50,8 +50,6 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v7
       - run: git config --global --add safe.directory $(pwd)
-      - name: Install libgit2 1.5 for git2go v34
-        run: backend/scripts/install-libgit2.sh
       - name: Install Go 1.27.3
         run: backend/scripts/install-go.sh
       - name: Build Python
@@ -94,8 +92,6 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v7
       - run: git config --global --add safe.directory $(pwd)
-      - name: Install libgit2 1.5 for git2go v34
-        run: backend/scripts/install-libgit2.sh
       - name: Install Go 1.27.3
         run: backend/scripts/install-go.sh
       - name: Build Python

.github/workflows/test.yml

@@ -47,8 +47,6 @@ jobs:
       uses: actions/checkout@v7
     - name: Set git config
       run: git config --global --add safe.directory $(pwd)
-    - name: Install libgit2 1.5 for git2go v34
-      run: backend/scripts/install-libgit2.sh
     - name: Install Go 1.27.3
       run: backend/scripts/install-go.sh
     - name: Check go.mod is tidy without generated mocks

.gitignore

@@ -147,7 +147,6 @@ NOTES.md
 main
 vendor
 bin
-libgit2
 .air.toml
 
 # Playwright CLI snapshots/logs (local dev)

AGENTS.md

@@ -129,4 +129,4 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/
 - Forgetting to add models to `GetTablesInfo()` fails `plugins/table_info_test.go`
 - Migration scripts must be added to `All()` in `register.go`
 - API changes require running `make swag` to update Swagger docs
-- Python plugins require `libgit2` for gitextractor functionality
+- `gitextractor` shells out to the `git` CLI; no git library or cgo is required

backend/DevelopmentManual.md

@@ -161,7 +161,7 @@ func TestDevlakeServer(t *testing.T) {
 		Plugins: []plugin.PluginMeta{ // list of plugins to make available to the server. Just create an empty instance of your intended plugin structs (conventionally declared in their respective `impl.go`)
 			github.Github{},
 			githubGraphql.GithubGraphql{},
-			gitextractor.GitExtractor{}, // this requires that your host machine has libgit2 installed, otherwise you'll get compilation errors
+			gitextractor.GitExtractor{},
 			gitlab.Gitlab(""),
 			jenkins.Jenkins{},
 			webhook.Webhook{},

backend/Dockerfile

@@ -23,14 +23,6 @@
 #While incubation status is not necessarily a reflection of the completeness or stability of the code,
 #it does indicate that the project has yet to be fully endorsed by the ASF.
 
-FROM --platform=linux/amd64 debian:bookworm as debian-amd64
-RUN apt-get update
-RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev
-
-FROM --platform=linux/arm64 debian:bookworm as debian-arm64
-RUN apt-get update
-RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev
-
 FROM --platform=$BUILDPLATFORM golang:1.27-bookworm as builder
 
 # docker build --build-arg GOPROXY=https://goproxy.io,direct -t mericodev/lake .
@@ -40,47 +32,11 @@ ARG HTTP_PROXY=
 ARG HTTPS_PROXY=
 
 RUN apt-get update
-RUN apt-get install -y gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev
-
-RUN if [ "$(arch)" != "aarch64" ] ; then \
-        apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu ; \
-    fi
-RUN if [ "$(arch)" != "x86_64" ] ; then \
-        apt-get install -y gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu ; \
-    fi
+RUN apt-get install -y ca-certificates git
 
 RUN go install github.com/vektra/mockery/v3@v3.8.1
 RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6
 
-COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include
-COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu
-COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu
-
-COPY --from=debian-arm64 /usr/include /rootfs-arm64/usr/include
-COPY --from=debian-arm64 /usr/lib/aarch64-linux-gnu /rootfs-arm64/usr/lib/aarch64-linux-gnu
-COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-gnu
-
-
-# set -e: otherwise a build failure for one arch is silently skipped and the
-# other arch's libgit2 is never installed under /usr/local/deps.
-RUN set -e; \
-    for arch in aarch64 x86_64 ; do \
-        mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \
-        wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \
-        cd libgit2-1.5.0 && \
-        mkdir build && cd build && \
-        if [ "$arch" = "aarch64" ] ; then \
-            cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
-                -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-arm64 \
-                -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \
-        elif [ "$arch" = "x86_64" ] ; then \
-            cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc \
-                -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-amd64 \
-                -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \
-        fi && \
-        make -j install ; \
-    done
-
 
 FROM builder as build
 
@@ -95,35 +51,20 @@ ARG GO_PLUGINS=
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
     if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
-        ln -s /usr/local/deps/aarch64 /usr/local/deps/target && \
-        export CC=aarch64-linux-gnu-gcc && \
-        export GOARCH=arm64 && \
-        export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-arm64/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/rootfs-arm64/lib/aarch64-linux-gnu" ; \
+        export GOARCH=arm64 ; \
     else \
-        ln -s /usr/local/deps/x86_64 /usr/local/deps/target && \
-        export CC=x86_64-linux-gnu-gcc && \
-        export GOARCH=amd64 && \
-        export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-amd64/usr/lib/x86_64-linux-gnu -Wl,-rpath-link,/rootfs-amd64/lib/x86_64-linux-gnu" ; \
+        export GOARCH=amd64 ; \
     fi && \
-    export PKG_CONFIG_PATH=/usr/local/deps/target/lib/pkgconfig && \
-    export CGO_ENABLED=1 &&\
+    export CGO_ENABLED=0 && \
     DEVLAKE_PLUGINS="$GO_PLUGINS" make all
 
-# remove symlink in lib, we will recreate in final image
-RUN cd /usr/local/deps/target/lib && \
-    for file in *.so* ; do \
-        if [ -L $file ] ; then \
-            unlink $file ; \
-        fi \
-    done
-
 
 FROM python:3.11-slim-bookworm as base
 
 # apt-get upgrade picks up Debian security fixes that the base tag does not contain yet
 RUN apt-get update && \
     apt-get upgrade -y && \
-    apt-get install -y python3-dev python3-pip tar pkg-config curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \
+    apt-get install -y python3-dev python3-pip tar pkg-config curl zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \
     apt-get clean && \
     rm -fr /usr/share/doc/* \
            /usr/share/info/* \
@@ -166,12 +107,6 @@ RUN /app/python/build.sh
 FROM base as devlake-base
 ARG DEBUG=
 
-# libraries
-ENV LD_LIBRARY_PATH=/app/libs
-RUN mkdir -p /app/libs
-COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs
-RUN ldconfig -vn /app/libs
-
 # apps
 # The single binary already contains every plugin; the .so files that
 # `make build-plugin` produces stay in the builder stage and are never copied.

backend/Dockerfile.local

@@ -31,21 +31,7 @@ ARG HTTPS_PROXY=
 
 RUN apt-get update && apt-get install -y \
     gcc \
-    binutils \
-    libfindbin-libs-perl \
-    cmake \
-    libssh2-1-dev \
-    libssl-dev \
-    zlib1g-dev
-
-# Install libgit2 for native platform only
-RUN mkdir -p /tmp/build && cd /tmp/build && \
-    wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \
-    cd libgit2-1.5.0 && \
-    mkdir build && cd build && \
-    cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \
-    make -j$(nproc) install && \
-    ldconfig
+    ca-certificates
 
 RUN go install github.com/vektra/mockery/v3@v3.8.1
 RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6
@@ -82,21 +68,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
 FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y \
-    libssh2-1 \
-    libssl3 \
     ca-certificates \
-    # gitextractor shells out to the git CLI unless
-    # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every
-    # clone fails with "git: executable file not found in $PATH".
+    # gitextractor shells out to the git CLI, so the binary must be present or
+    # every clone fails with "git: executable file not found in $PATH".
     # debian:bookworm-slim does not ship it; the official image inherits it from
     # its python base.
     git \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy libgit2
-COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/
-RUN ldconfig
-
 WORKDIR /app
 
 COPY --from=builder /app/bin/lake ./bin/lake

backend/go.mod

@@ -13,15 +13,13 @@ require (
 	github.com/coreos/go-oidc/v3 v3.21.0
 	// github.com/gin-contrib/cors v1.6.0
 	github.com/gin-gonic/gin v1.12.0
-	github.com/go-git/go-git/v5 v5.19.3
 	github.com/go-playground/validator/v10 v10.30.5
 	github.com/go-sql-driver/mysql v1.10.1
 	github.com/go-viper/mapstructure/v2 v2.5.0
 	github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.11.0
-	github.com/libgit2/git2go/v34 v34.0.0
 	github.com/magiconair/properties v1.18.12
 	github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac
 	github.com/panjf2000/ants/v2 v2.12.1
@@ -48,11 +46,8 @@ require (
 )
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/go-crypto v1.1.6 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 // indirect
@@ -72,19 +67,14 @@ require (
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic v1.15.0 // indirect
 	github.com/bytedance/sonic/loader v0.5.0 // indirect
-	github.com/cloudflare/circl v1.6.3 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
-	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.15 // indirect
 	github.com/getsentry/sentry-go v0.46.0 // indirect
 	github.com/gin-contrib/sse v1.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/go-billy/v5 v5.9.2 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
@@ -95,17 +85,14 @@ require (
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/goccy/go-yaml v1.19.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -115,13 +102,10 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pjbgf/sha1cd v0.7.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/quic-go/quic-go v0.59.1 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
-	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
-	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
@@ -131,7 +115,6 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.3.1 // indirect
-	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/arch v0.22.0 // indirect
@@ -141,6 +124,5 @@ require (
 	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/tools v0.51.0 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
-	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )