diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b7099ace08b..e795825084f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,10 +15,7 @@ FROM mcr.microsoft.com/devcontainers/go:2-1.27-bookworm -ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib:/usr/local/lib/pkgconfig -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ENV DEVLAKE_PLUGINS=argocd,bamboo,bitbucket,circleci,customize,dora,gitextractor,github,github_graphql,gitlab,issue_trace,jenkins,jira,org,pagerduty,refdiff,slack,sonarqube,trello,webhook RUN apt-get update -y -RUN apt-get install pkg-config python3-dev default-libmysqlclient-dev build-essential libpq-dev cmake -y -RUN git clone -b v1.5.0 https://github.com/libgit2/libgit2.git && cd libgit2 && mkdir build && cd build && cmake .. && make && make install +RUN apt-get install python3-dev default-libmysqlclient-dev build-essential libpq-dev -y diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 41f2b2a725b..aaf59c2d52a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -58,9 +58,6 @@ updates: # The Go toolchain directive is coupled to the CI bootstrap version and # to the golang base images; it is raised deliberately, never by a bot. - dependency-name: go - # git2go is ABI-coupled to the libgit2 version installed in the builder - # image. A mismatch fails at link time, so this is always manual. - - dependency-name: github.com/libgit2/git2go/* # swag v2 is still a release candidate and requires regenerating every # swagger annotation. Tracked as an externally gated upgrade. - dependency-name: github.com/swaggo/swag diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 0df4d8308bc..a21ca46c31c 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -32,8 +32,6 @@ jobs: - uses: actions/checkout@v7 - name: Set git config run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Install mockery 3.8.1 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 3499df44ed0..1512e7273ae 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -50,8 +50,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Build Python @@ -94,8 +92,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Build Python diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86e2ea95e32..6c0ac07079c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,8 +47,6 @@ jobs: uses: actions/checkout@v7 - name: Set git config run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Check go.mod is tidy without generated mocks diff --git a/.gitignore b/.gitignore index 0877343e6bc..7097c0187d1 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,6 @@ NOTES.md main vendor bin -libgit2 .air.toml # Playwright CLI snapshots/logs (local dev) diff --git a/AGENTS.md b/AGENTS.md index b896b7af69c..f558e08db3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,4 +129,4 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/ - Forgetting to add models to `GetTablesInfo()` fails `plugins/table_info_test.go` - Migration scripts must be added to `All()` in `register.go` - API changes require running `make swag` to update Swagger docs -- Python plugins require `libgit2` for gitextractor functionality +- `gitextractor` shells out to the `git` CLI; no git library or cgo is required diff --git a/backend/DevelopmentManual.md b/backend/DevelopmentManual.md index 26208d55ff5..96ba60958db 100644 --- a/backend/DevelopmentManual.md +++ b/backend/DevelopmentManual.md @@ -161,7 +161,7 @@ func TestDevlakeServer(t *testing.T) { Plugins: []plugin.PluginMeta{ // list of plugins to make available to the server. Just create an empty instance of your intended plugin structs (conventionally declared in their respective `impl.go`) github.Github{}, githubGraphql.GithubGraphql{}, - gitextractor.GitExtractor{}, // this requires that your host machine has libgit2 installed, otherwise you'll get compilation errors + gitextractor.GitExtractor{}, gitlab.Gitlab(""), jenkins.Jenkins{}, webhook.Webhook{}, diff --git a/backend/Dockerfile b/backend/Dockerfile index 877a1ac7b88..9ed02234a0a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -23,14 +23,6 @@ #While incubation status is not necessarily a reflection of the completeness or stability of the code, #it does indicate that the project has yet to be fully endorsed by the ASF. -FROM --platform=linux/amd64 debian:bookworm as debian-amd64 -RUN apt-get update -RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - -FROM --platform=linux/arm64 debian:bookworm as debian-arm64 -RUN apt-get update -RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - FROM --platform=$BUILDPLATFORM golang:1.27-bookworm as builder # docker build --build-arg GOPROXY=https://goproxy.io,direct -t mericodev/lake . @@ -40,47 +32,11 @@ ARG HTTP_PROXY= ARG HTTPS_PROXY= RUN apt-get update -RUN apt-get install -y gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev - -RUN if [ "$(arch)" != "aarch64" ] ; then \ - apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu ; \ - fi -RUN if [ "$(arch)" != "x86_64" ] ; then \ - apt-get install -y gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu ; \ - fi +RUN apt-get install -y ca-certificates git RUN go install github.com/vektra/mockery/v3@v3.8.1 RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 -COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include -COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu -COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu - -COPY --from=debian-arm64 /usr/include /rootfs-arm64/usr/include -COPY --from=debian-arm64 /usr/lib/aarch64-linux-gnu /rootfs-arm64/usr/lib/aarch64-linux-gnu -COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-gnu - - -# set -e: otherwise a build failure for one arch is silently skipped and the -# other arch's libgit2 is never installed under /usr/local/deps. -RUN set -e; \ - for arch in aarch64 x86_64 ; do \ - mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - if [ "$arch" = "aarch64" ] ; then \ - cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \ - -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-arm64 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \ - elif [ "$arch" = "x86_64" ] ; then \ - cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc \ - -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-amd64 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \ - fi && \ - make -j install ; \ - done - FROM builder as build @@ -95,35 +51,20 @@ ARG GO_PLUGINS= RUN --mount=type=cache,target=/root/.cache/go-build \ if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \ - ln -s /usr/local/deps/aarch64 /usr/local/deps/target && \ - export CC=aarch64-linux-gnu-gcc && \ - export GOARCH=arm64 && \ - export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-arm64/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/rootfs-arm64/lib/aarch64-linux-gnu" ; \ + export GOARCH=arm64 ; \ else \ - ln -s /usr/local/deps/x86_64 /usr/local/deps/target && \ - export CC=x86_64-linux-gnu-gcc && \ - export GOARCH=amd64 && \ - export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-amd64/usr/lib/x86_64-linux-gnu -Wl,-rpath-link,/rootfs-amd64/lib/x86_64-linux-gnu" ; \ + export GOARCH=amd64 ; \ fi && \ - export PKG_CONFIG_PATH=/usr/local/deps/target/lib/pkgconfig && \ - export CGO_ENABLED=1 &&\ + export CGO_ENABLED=0 && \ DEVLAKE_PLUGINS="$GO_PLUGINS" make all -# remove symlink in lib, we will recreate in final image -RUN cd /usr/local/deps/target/lib && \ - for file in *.so* ; do \ - if [ -L $file ] ; then \ - unlink $file ; \ - fi \ - done - FROM python:3.11-slim-bookworm as base # apt-get upgrade picks up Debian security fixes that the base tag does not contain yet RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y python3-dev python3-pip tar pkg-config curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ + apt-get install -y python3-dev python3-pip tar pkg-config curl zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ apt-get clean && \ rm -fr /usr/share/doc/* \ /usr/share/info/* \ @@ -166,12 +107,6 @@ RUN /app/python/build.sh FROM base as devlake-base ARG DEBUG= -# libraries -ENV LD_LIBRARY_PATH=/app/libs -RUN mkdir -p /app/libs -COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs -RUN ldconfig -vn /app/libs - # apps # The single binary already contains every plugin; the .so files that # `make build-plugin` produces stay in the builder stage and are never copied. diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index e2046db3bd2..bd9a16cf27a 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -31,21 +31,7 @@ ARG HTTPS_PROXY= RUN apt-get update && apt-get install -y \ gcc \ - binutils \ - libfindbin-libs-perl \ - cmake \ - libssh2-1-dev \ - libssl-dev \ - zlib1g-dev - -# Install libgit2 for native platform only -RUN mkdir -p /tmp/build && cd /tmp/build && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \ - make -j$(nproc) install && \ - ldconfig + ca-certificates RUN go install github.com/vektra/mockery/v3@v3.8.1 RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 @@ -82,21 +68,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y \ - libssh2-1 \ - libssl3 \ ca-certificates \ - # gitextractor shells out to the git CLI unless - # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every - # clone fails with "git: executable file not found in $PATH". + # gitextractor shells out to the git CLI, so the binary must be present or + # every clone fails with "git: executable file not found in $PATH". # debian:bookworm-slim does not ship it; the official image inherits it from # its python base. git \ && rm -rf /var/lib/apt/lists/* -# Copy libgit2 -COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/ -RUN ldconfig - WORKDIR /app COPY --from=builder /app/bin/lake ./bin/lake diff --git a/backend/go.mod b/backend/go.mod index f1f382839f3..fa0b0cbc5da 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -13,7 +13,6 @@ require ( github.com/coreos/go-oidc/v3 v3.21.0 // github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.12.0 - github.com/go-git/go-git/v5 v5.19.3 github.com/go-playground/validator/v10 v10.30.5 github.com/go-sql-driver/mysql v1.10.1 github.com/go-viper/mapstructure/v2 v2.5.0 @@ -21,7 +20,6 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 - github.com/libgit2/git2go/v34 v34.0.0 github.com/magiconair/properties v1.18.12 github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac github.com/panjf2000/ants/v2 v2.12.1 @@ -48,11 +46,8 @@ require ( ) require ( - dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 // indirect @@ -72,19 +67,14 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect - github.com/cloudflare/circl v1.6.3 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cyphar/filepath-securejoin v0.6.1 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.15 // indirect github.com/getsentry/sentry-go v0.46.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.9.2 // indirect github.com/go-jose/go-jose/v4 v4.1.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -95,17 +85,14 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -115,13 +102,10 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pjbgf/sha1cd v0.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -131,7 +115,6 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.22.0 // indirect @@ -141,6 +124,5 @@ require ( golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.51.0 // indirect google.golang.org/protobuf v1.36.12 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index c6a7886bf45..7e7964921b4 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,22 +1,11 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= -github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0= github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 h1:t7FxqJ5KHDeKceyD4HtGaG6z7hMSeVGjs5/4c+L5ArM= @@ -63,8 +52,6 @@ github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiD github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cockroachdb/errors v1.14.0 h1:EfdVEJpN3z8rPMo43Yit59LxoiIa470fSXpZXuEs+ZI= @@ -77,15 +64,9 @@ github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBq github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= -github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= -github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -100,18 +81,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= -github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.19.3 h1:qHttbZ+Am7wEjdTpR1XMQ4rl42FK9SIXzZ4o9x7s6M4= -github.com/go-git/go-git/v5 v5.19.3/go.mod h1:Ye4C8dVigkqrv9UsB4NMdSLV4yAIkLiIhW61gcOyhl4= github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -150,13 +121,9 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -169,8 +136,6 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -179,8 +144,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -194,8 +157,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= -github.com/libgit2/git2go/v34 v34.0.0 h1:UKoUaKLmiCRbOCD3PtUi2hD6hESSXzME/9OUZrGcgu8= -github.com/libgit2/git2go/v34 v34.0.0/go.mod h1:blVco2jDAw6YTXkErMMqzHLcAjKkwF0aWIRHBqiJkZ0= github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I= github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -217,16 +178,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10= github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pjbgf/sha1cd v0.7.0 h1:ZRNPKHj+gfkLBf0KJv/p1Hmz+7bqCT5o311YX0nq+DA= -github.com/pjbgf/sha1cd v0.7.0/go.mod h1:pKR5Li+qTCo+ebqITZfwPlJGoCFCvSO00qxjbNtTUFQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -243,13 +200,8 @@ github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52Z github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -269,9 +221,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -300,8 +250,6 @@ github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/viant/afs v1.30.1 h1:l+cIg7pVo5Cdia2GqaK+h3hwvj26QHg6VHmuatZ0CTM= github.com/viant/afs v1.30.1/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -319,9 +267,7 @@ golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= @@ -337,7 +283,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= @@ -352,22 +297,16 @@ golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -395,18 +334,14 @@ google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fw google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk= gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY= diff --git a/backend/plugins/gitextractor/README.md b/backend/plugins/gitextractor/README.md index 9d70677828a..e1e0266971b 100644 --- a/backend/plugins/gitextractor/README.md +++ b/backend/plugins/gitextractor/README.md @@ -16,8 +16,9 @@ limitations under the License. --> Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/gitextractor) +The plugin reads repositories with the `git` command line, so `git` (2.31 or +newer) must be installed where the server runs. No libgit2 or cgo is required. -Build with local compiled v1.5.0 libgit2 ``` -CGO_LDFLAGS="-L/usr/local/lib -lgit2" CGO_CFLAGS="-I/usr/local/include" go build ./plugins/gitextractor/... 2>&1 | tail -5 +go build ./plugins/gitextractor/... ``` \ No newline at end of file diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go index 67e0b7f0f2a..da8f9b9ca0c 100644 --- a/backend/plugins/gitextractor/gitextractor.go +++ b/backend/plugins/gitextractor/gitextractor.go @@ -35,7 +35,6 @@ func main() { // pk := cmd.Flags().StringP("privateKey", "k", "", "private key file") // pkPass := cmd.Flags().StringP("privateKeyPassPhrase", "P", "", "passphrase for private key") proxy := cmd.Flags().StringP("proxy", "x", "", "proxy") - useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2") skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", false, "") skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "") noShallowClone := cmd.Flags().BoolP("noShallowClone", "A", false, "") @@ -52,7 +51,6 @@ func main() { // "privateKey": * // "passphrase" "proxy": *proxy, - "useGoGit": *useGoGit, "skipCommitStat": skipCommitStat, "skipCommitFiles": skipCommitFiles, "noShallowClone": noShallowClone, diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index d0f763489af..6b0aa1bc5e9 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -115,10 +115,8 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st } *optValue = &defValue } - loadBool(&op.UseGoGit, "UseGoGit", false) loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", false) loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true) - log.Info("UseGoGit: %v", *op.UseGoGit) log.Info("SkipCommitStat: %v", *op.SkipCommitStat) log.Info("SkipCommitFiles: %v", *op.SkipCommitFiles) diff --git a/backend/plugins/gitextractor/models/diffline.go b/backend/plugins/gitextractor/models/diffline.go deleted file mode 100644 index 5375df29417..00000000000 --- a/backend/plugins/gitextractor/models/diffline.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package models - -import git "github.com/libgit2/git2go/v34" - -type DiffLines []git.DiffLine - -// Len return length of diffLines -func (diffLines DiffLines) Len() int { - return len(diffLines) -} - -// Less function -func (diffLines DiffLines) Less(i, j int) bool { - return diffLines[i].OldLineno > diffLines[j].OldLineno -} - -// Swap function -func (diffLines DiffLines) Swap(i, j int) { - temp := diffLines[i] - diffLines[i] = diffLines[j] - diffLines[j] = temp -} diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 82fc9d57afb..06f1a7acb00 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -46,7 +46,6 @@ const maxExtraDeepenRounds = 10 // CloneRepoConfig is the configuration for the CloneRepo method // the subtask should run in Full Sync mode whenever the configuration is changed type CloneRepoConfig struct { - UseGoGit *bool SkipCommitStat *bool SkipCommitFiles *bool NoShallowClone bool @@ -72,7 +71,6 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, SubTaskContext: ctx, Params: taskData.Options.GitExtractorApiParams, SubtaskConfig: CloneRepoConfig{ - UseGoGit: taskData.Options.UseGoGit, SkipCommitStat: taskData.Options.SkipCommitStat, SkipCommitFiles: taskData.Options.SkipCommitFiles, NoShallowClone: taskData.Options.NoShallowClone, @@ -95,7 +93,7 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, func (g *GitcliCloner) prepareSync() errors.Error { taskData := g.taskData // Disable automatic (detached) gc/maintenance after clone/fetch. Otherwise git may - // repack the repo in the background while libgit2 is already reading it, which + // repack the repo in the background while the collector is reading it, which // makes temporary pack files (.tmp--pack-*.idx) vanish and the collection fail. g.globalConfigArgs = append(g.globalConfigArgs, "-c", "gc.auto=0", @@ -201,7 +199,7 @@ func (g *GitcliCloner) CloneRepo() errors.Error { if g.taskData.Options.NoShallowClone { // data source does not support shallow clone // 1. perform a full clone to accommodate - // 2. perform a local shallow clone to reduce the libgit2 memory usage + // 2. perform a local shallow clone so the collector reads a smaller repo if err := g.doubleClone(); err != nil { return err } diff --git a/backend/plugins/gitextractor/parser/clone_gitcli_test.go b/backend/plugins/gitextractor/parser/clone_gitcli_test.go index 790980f64f6..83636b97e90 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli_test.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli_test.go @@ -127,7 +127,7 @@ func TestIncrementalCloneKeepsCommitsBehindMergeBoundary(t *testing.T) { } // git fetch may start `git maintenance run --auto` / `git gc --auto` in the -// background. That repacks the repo while libgit2 is already reading it, so +// background. That repacks the repo while the collector is reading it, so // temporary pack files (.tmp--pack-*.idx) vanish under its feet and the // collection fails with "could not find ... to open". func TestCloneDoesNotStartAutoMaintenance(t *testing.T) { diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go new file mode 100644 index 00000000000..c2029961b0f --- /dev/null +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -0,0 +1,655 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "context" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer/code" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + mockplugin "github.com/apache/devlake/mocks/core/plugin" + "github.com/apache/devlake/plugins/gitextractor/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// These tests describe the collector contract: what the git command line +// collector writes for a known repository. The fixtures are built with the git +// CLI itself, so the tests do not depend on any git library. + +func newTestCollector(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector +} + +func forEachCollector(t *testing.T, fn func(t *testing.T)) { + t.Helper() + t.Run("gitcli", fn) +} + +// recordStore is a models.Store that records everything the collectors write. +type recordStore struct { + incremental bool + closed bool + repoCommits []*code.RepoCommit + commits []*code.Commit + refs []*code.Ref + commitFiles []*code.CommitFile + parents []*code.CommitParent + components []*code.CommitFileComponent + lineChanges []*code.CommitLineChange + snapshots []*code.RepoSnapshot +} + +var _ models.Store = (*recordStore)(nil) + +func (s *recordStore) SetIncrementalMode(v bool) { s.incremental = v } + +func (s *recordStore) RepoCommits(v *code.RepoCommit) errors.Error { + s.repoCommits = append(s.repoCommits, v) + return nil +} + +func (s *recordStore) Commits(v *code.Commit) errors.Error { + cp := *v + s.commits = append(s.commits, &cp) + return nil +} + +func (s *recordStore) Refs(v *code.Ref) errors.Error { + cp := *v + s.refs = append(s.refs, &cp) + return nil +} + +func (s *recordStore) CommitFiles(v *code.CommitFile) errors.Error { + cp := *v + s.commitFiles = append(s.commitFiles, &cp) + return nil +} + +func (s *recordStore) CommitParents(v []*code.CommitParent) errors.Error { + s.parents = append(s.parents, v...) + return nil +} + +func (s *recordStore) CommitFileComponents(v *code.CommitFileComponent) errors.Error { + cp := *v + s.components = append(s.components, &cp) + return nil +} + +func (s *recordStore) CommitLineChange(v *code.CommitLineChange) errors.Error { + cp := *v + s.lineChanges = append(s.lineChanges, &cp) + return nil +} + +func (s *recordStore) RepoSnapshot(v *code.RepoSnapshot) errors.Error { + cp := *v + s.snapshots = append(s.snapshots, &cp) + return nil +} + +func (s *recordStore) Close() errors.Error { + s.closed = true + return nil +} + +func boolPtr(v bool) *bool { return &v } + +func defaultOptions(repoId string) *GitExtractorOptions { + return &GitExtractorOptions{ + GitExtractorApiParams: GitExtractorApiParams{RepoId: repoId}, + SkipCommitStat: boolPtr(false), + SkipCommitFiles: boolPtr(false), + } +} + +// newCollectorContext returns a subtask context backed by a mocked DAL. The +// components are returned from Dal.All, which the collectors use to map files +// to components. +func newCollectorContext(t *testing.T, opts *GitExtractorOptions, components []code.Component) *mockplugin.SubTaskContext { + t.Helper() + dalMock := new(mockdal.Dal) + dalMock.On("All", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + if len(components) == 0 { + return + } + dst, ok := args.Get(0).(*[]code.Component) + if !ok { + return + } + *dst = append(*dst, components...) + }) + dalMock.On("Delete", mock.Anything, mock.Anything).Return(nil) + + ctx := unithelper.DummySubTaskContext(dalMock) + ctx.On("GetContext").Return(context.Background()) + ctx.On("GetData").Return(&GitExtractorTaskData{Options: opts}) + return ctx +} + +func commitsBySha(commits []*code.Commit) map[string]*code.Commit { + out := make(map[string]*code.Commit, len(commits)) + for _, commit := range commits { + out[commit.Sha] = commit + } + return out +} + +func refsByName(refs []*code.Ref) map[string]*code.Ref { + out := make(map[string]*code.Ref, len(refs)) + for _, ref := range refs { + out[ref.Name] = ref + } + return out +} + +func componentsById(components []*code.CommitFileComponent) map[string]string { + out := make(map[string]string, len(components)) + for _, component := range components { + out[component.CommitFileId] = component.ComponentName + } + return out +} + +// filesByCommit returns commit sha -> file path -> [additions, deletions]. +func filesByCommit(files []*code.CommitFile) map[string]map[string][2]int { + out := make(map[string]map[string][2]int) + for _, file := range files { + if out[file.CommitSha] == nil { + out[file.CommitSha] = make(map[string][2]int) + } + out[file.CommitSha][file.FilePath] = [2]int{file.Additions, file.Deletions} + } + return out +} + +func parentShas(parents []*code.CommitParent, commitSha string) []string { + var out []string + for _, parent := range parents { + if parent.CommitSha == commitSha { + out = append(out, parent.ParentCommitSha) + } + } + sort.Strings(out) + return out +} + +func TestCollectorCountsAndRefs(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + commits, err := collector.CountCommits(context.Background()) + require.NoError(t, err) + assert.Equal(t, 5, commits) + + tags, err := collector.CountTags(context.Background()) + require.NoError(t, err) + assert.Equal(t, 2, tags) + // HEAD points at main, which is not counted. + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, branches) + + assert.NoError(t, collector.CollectTags(ctx)) + assert.NoError(t, collector.CollectBranches(ctx)) + + refs := refsByName(store.refs) + require.Len(t, refs, 4) + + // Tag names are stored as the full ref name. + light := refs["refs/tags/v1.0.0"] + require.NotNil(t, light, "lightweight tag v1.0.0") + assert.Equal(t, "test-repo:refs/tags/v1.0.0", light.Id) + assert.Equal(t, repo.C1, light.CommitSha) + assert.Equal(t, TAG, light.RefType) + assert.Equal(t, "test-repo", light.RepoId) + + annotated := refs["refs/tags/v2.0.0"] + require.NotNil(t, annotated, "annotated tag v2.0.0 should resolve to its target commit") + assert.Equal(t, repo.C3, annotated.CommitSha) + assert.Equal(t, TAG, annotated.RefType) + + mainRef := refs["main"] + require.NotNil(t, mainRef) + assert.Equal(t, repo.C5, mainRef.CommitSha) + assert.Equal(t, BRANCH, mainRef.RefType) + assert.True(t, mainRef.IsDefault) + + featureRef := refs["feature"] + require.NotNil(t, featureRef) + assert.Equal(t, repo.C2, featureRef.CommitSha) + assert.Equal(t, BRANCH, featureRef.RefType) + assert.False(t, featureRef.IsDefault) + }) +} + +func TestCollectorCommits(t *testing.T) { + repo := buildCanonicalRepo(t) + repoId := "test-repo" + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, repoId, store) + ctx := newCollectorContext(t, defaultOptions(repoId), nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + require.Len(t, commits, 5) + require.Len(t, store.repoCommits, 5) + + for _, tc := range []struct { + sha string + when string + msg string + }{ + {repo.C1, "2026-01-01T00:00:00Z", "c1: initial import"}, + {repo.C2, "2026-01-01T01:00:00Z", "c2: add feature file"}, + {repo.C3, "2026-01-01T02:00:00Z", "c3: reword and extend a.txt"}, + {repo.C4, "2026-01-01T03:00:00Z", "c4: mode and binary change"}, + {repo.C5, "2026-01-01T04:00:00Z", "c5: merge feature"}, + } { + commit := commits[tc.sha] + require.NotNil(t, commit, tc.msg) + assert.Equal(t, tc.msg+"\n", commit.Message) + assert.Equal(t, testAuthorName, commit.AuthorName) + assert.Equal(t, testAuthorEmail, commit.AuthorEmail) + assert.Equal(t, testAuthorEmail, commit.AuthorId) + assert.Equal(t, testAuthorName, commit.CommitterName) + assert.Equal(t, testAuthorEmail, commit.CommitterEmail) + assert.Equal(t, testAuthorEmail, commit.CommitterId) + assert.Equal(t, tc.when, commit.AuthoredDate.UTC().Format("2006-01-02T15:04:05Z")) + assert.Equal(t, tc.when, commit.CommittedDate.UTC().Format("2006-01-02T15:04:05Z")) + } + + assert.Empty(t, parentShas(store.parents, repo.C1)) + assert.Equal(t, []string{repo.C1}, parentShas(store.parents, repo.C2)) + assert.Equal(t, []string{repo.C1}, parentShas(store.parents, repo.C3)) + assert.Equal(t, []string{repo.C3}, parentShas(store.parents, repo.C4)) + assert.ElementsMatch(t, []string{repo.C2, repo.C4}, parentShas(store.parents, repo.C5)) + + files := filesByCommit(store.commitFiles) + expectedFiles := map[string]map[string][2]int{ + repo.C1: { + "a.txt": {3, 0}, + "bin.dat": {0, 0}, + "dir with space/b.txt": {1, 0}, + "empty.txt": {0, 0}, + }, + repo.C2: {"feature.txt": {1, 0}}, + repo.C3: { + "a.txt": {2, 1}, + "empty.txt": {0, 0}, + }, + repo.C4: { + "a.txt": {0, 0}, + "bin.dat": {0, 0}, + }, + // The merge is compared against its first parent, so only the file + // the merge brought in is reported. + repo.C5: {"feature.txt": {1, 0}}, + } + for sha, want := range expectedFiles { + assert.Equal(t, want, files[sha], sha) + } + + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 0, commits[repo.C1].Deletions) + assert.Equal(t, 1, commits[repo.C2].Additions) + assert.Equal(t, 2, commits[repo.C3].Additions) + assert.Equal(t, 1, commits[repo.C3].Deletions) + assert.Equal(t, 0, commits[repo.C4].Additions) + assert.Equal(t, 0, commits[repo.C4].Deletions) + assert.Equal(t, 1, commits[repo.C5].Additions) + assert.Equal(t, 0, commits[repo.C5].Deletions) + + for _, file := range store.commitFiles { + assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) + } + // This repository has no components configured, so every file would + // only get a meaningless "Default" component row. + assert.Empty(t, store.components) + }) +} + +func TestCollectorUsesConfiguredComponents(t *testing.T) { + repo := buildLinearRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), []code.Component{ + {RepoId: "test-repo", Name: "docs", PathRegex: `^c\.txt$`}, + }) + + require.NoError(t, collector.CollectCommits(ctx)) + + components := componentsById(store.components) + files := filesByCommit(store.commitFiles)[repo.C4] + require.Contains(t, files, "c.txt") + assert.Equal(t, "docs", components[genCommitFileId(repo.C4, "c.txt")]) + assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) + }) +} + +func TestCollectorSkipCommitStat(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.SkipCommitStat = boolPtr(true) + opts.SkipCommitFiles = boolPtr(true) + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + require.Len(t, store.commits, 5) + for _, commit := range store.commits { + assert.Zero(t, commit.Additions) + assert.Zero(t, commit.Deletions) + } + assert.Empty(t, store.commitFiles) + assert.Empty(t, store.components) + }) +} + +func TestCollectorSkipCommitFiles(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.SkipCommitFiles = boolPtr(true) + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + assert.Empty(t, store.commitFiles) + assert.Empty(t, store.components) + commits := commitsBySha(store.commits) + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 1, commits[repo.C2].Additions) + }) +} + +func TestCollectorExcludeFileExtensions(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.ExcludeFileExtensions = []string{".DAT"} + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 0, commits[repo.C4].Additions) + assert.Equal(t, 0, commits[repo.C4].Deletions) + + files := filesByCommit(store.commitFiles) + assert.NotContains(t, files[repo.C1], "bin.dat") + assert.NotContains(t, files[repo.C4], "bin.dat") + assert.Contains(t, files[repo.C1], "a.txt") + }) +} + +func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + require.NotEqual(t, c1, c2) + + dir := f.shallowBareClone() + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + // The tip is a shallow boundary: its parent is not in the object + // database, so the collector must not compute stats for it (a full-tree + // diff would store an absurdly large addition count). + require.NoError(t, collector.CollectCommits(ctx)) + assert.Empty(t, store.commits) + assert.Empty(t, store.commitFiles) + }) +} + +func TestCollectorSkipsShallowBoundaryEvenWhenItsParentExists(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + // keep the parent reachable through another branch + f.git("branch", "old", c1) + + dir := f.bareClone() + // Mark the tip as a shallow boundary although its parent object is still + // in the object database. git then renders the tip as a root commit, so + // the collector must skip it based on the shallow file, not on the parent + // being missing. + require.NoError(t, os.WriteFile(filepath.Join(dir, "shallow"), []byte(c2+"\n"), 0o644)) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + assert.NotContains(t, commits, c2, "the shallow boundary must not be collected") + require.Contains(t, commits, c1) + assert.Equal(t, 1, commits[c1].Additions) + }) +} + +func TestCollectorEmptyRepo(t *testing.T) { + dir := newGitFixture(t).bareClone() + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + commits, err := collector.CountCommits(context.Background()) + require.NoError(t, err) + assert.Zero(t, commits) + tags, err := collector.CountTags(context.Background()) + require.NoError(t, err) + assert.Zero(t, tags) + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Zero(t, branches) + + assert.NoError(t, collector.CollectTags(ctx)) + assert.NoError(t, collector.CollectBranches(ctx)) + assert.NoError(t, collector.CollectCommits(ctx)) + assert.Empty(t, store.refs) + assert.Empty(t, store.commits) + }) +} + +func TestCollectorHonoursCancelledContext(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := collector.CountCommits(ctx) + assert.ErrorIs(t, err, context.Canceled) + _, err = collector.CountTags(ctx) + assert.ErrorIs(t, err, context.Canceled) + _, err = collector.CountBranches(ctx) + assert.ErrorIs(t, err, context.Canceled) + }) +} + +func TestCollectorRejectsNonRepo(t *testing.T) { + _, err := NewGitcliRepoCollector(t.TempDir(), "test-repo", nil, unithelper.DummyLogger()) + assert.Error(t, err, "the collector should reject a non-repository") +} + +func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.checkout("feature", true) + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + require.NotEqual(t, c1, c2) + dir := f.bareClone() + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, dir, "test-repo", store) + + // HEAD points at feature, which is not counted but is the default ref. + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, branches) + + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + require.NoError(t, collector.CollectBranches(ctx)) + + refs := refsByName(store.refs) + require.Len(t, refs, 2) + assert.True(t, refs["feature"].IsDefault) + assert.False(t, refs["main"].IsDefault) + }) +} + +// TestCollectorDiffLineSnapshot checks the repo snapshot against git blame, +// which is an independent oracle for "which commit introduced each line". +func TestCollectorDiffLineSnapshot(t *testing.T) { + repo := buildLinearRepo(t) + expected := blameAtHEAD(t, repo.Dir) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectDiffLine(ctx)) + + got := make(map[string]map[int]string) + for _, snapshot := range store.snapshots { + if got[snapshot.FilePath] == nil { + got[snapshot.FilePath] = make(map[int]string) + } + got[snapshot.FilePath][snapshot.LineNo] = snapshot.CommitSha + } + + // b.txt is deleted in c4, so it has no lines in HEAD and no snapshot rows. + assert.NotContains(t, got, "b.txt") + for file, lines := range expected { + assert.Equal(t, lines, got[file], file) + } + + assert.NotEmpty(t, store.lineChanges) + }) +} + +func TestCollectorCollectAllAndClose(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + cleanedUp := false + require.NoError(t, collector.SetCleanUp(func() { cleanedUp = true })) + + require.NoError(t, collector.CollectAll(ctx)) + assert.Len(t, store.commits, 5) + assert.Len(t, store.repoCommits, 5) + assert.Len(t, store.refs, 4) + assert.NotEmpty(t, store.commitFiles) + assert.NotEmpty(t, store.snapshots) + assert.NotEmpty(t, store.lineChanges) + + require.NoError(t, collector.Close(context.Background())) + assert.True(t, store.closed) + assert.True(t, cleanedUp) + }) +} + +func TestCollectorCommitLineChanges(t *testing.T) { + repo := buildLinearRepo(t) + + store := &recordStore{} + collector := newTestCollector(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectDiffLine(ctx)) + require.NotEmpty(t, store.lineChanges) + + // c3 appends line 4 to a.txt, so a.txt gets one addition at new line 4. + var additions []*code.CommitLineChange + for _, change := range store.lineChanges { + if change.CommitSha == repo.C3 && change.NewFilePath == "a.txt" && change.ChangedType == "Addition" { + additions = append(additions, change) + } + } + require.Len(t, additions, 1) + assert.Equal(t, 4, additions[0].LineNoNew) + assert.Equal(t, -1, additions[0].LineNoOld) + assert.Equal(t, 1, additions[0].HunkNum) + assert.Equal(t, "", additions[0].PrevCommit) + assert.Equal(t, repo.C3+":a.txt:-1:4", additions[0].Id) + + // c2 rewrites line 2 of a.txt; the deletion must be attributed to c1. + var deletions []*code.CommitLineChange + for _, change := range store.lineChanges { + if change.CommitSha == repo.C2 && change.NewFilePath == "a.txt" && change.ChangedType == "Deletion" { + deletions = append(deletions, change) + } + } + require.Len(t, deletions, 1) + assert.Equal(t, 2, deletions[0].LineNoOld) + assert.Equal(t, repo.C1, deletions[0].PrevCommit) +} diff --git a/backend/plugins/gitextractor/parser/commitparse.go b/backend/plugins/gitextractor/parser/commitparse.go new file mode 100644 index 00000000000..4f52f89f824 --- /dev/null +++ b/backend/plugins/gitextractor/parser/commitparse.go @@ -0,0 +1,172 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +// parsedCommit is the metadata of one commit object, as stored by git. +type parsedCommit struct { + Sha string + Parents []string + Author parsedSignature + Committer parsedSignature + Message string +} + +type parsedSignature struct { + Name string + Email string + When time.Time +} + +// parseCommitObject parses the raw content of a commit object. +func parseCommitObject(sha string, content []byte) (*parsedCommit, error) { + commit := &parsedCommit{Sha: sha} + + header := content + if idx := bytes.Index(content, []byte("\n\n")); idx >= 0 { + header = content[:idx] + commit.Message = string(content[idx+2:]) + } + + for _, line := range strings.Split(string(header), "\n") { + if line == "" { + continue + } + if line[0] == ' ' { + // continuation of a multi-line header such as gpgsig, not used here + continue + } + key, value, ok := strings.Cut(line, " ") + if !ok { + continue + } + switch key { + case "parent": + commit.Parents = append(commit.Parents, value) + case "author": + signature, err := parseSignature(value) + if err != nil { + return nil, fmt.Errorf("commit %s: %w", sha, err) + } + commit.Author = signature + case "committer": + signature, err := parseSignature(value) + if err != nil { + return nil, fmt.Errorf("commit %s: %w", sha, err) + } + commit.Committer = signature + } + } + return commit, nil +} + +// parseSignature parses "Name ". +func parseSignature(value string) (parsedSignature, error) { + var signature parsedSignature + + tzSpace := strings.LastIndexByte(value, ' ') + if tzSpace < 0 { + return signature, fmt.Errorf("invalid signature %q", value) + } + timezone := value[tzSpace+1:] + rest := value[:tzSpace] + + tsSpace := strings.LastIndexByte(rest, ' ') + if tsSpace < 0 { + return signature, fmt.Errorf("invalid signature %q", value) + } + seconds, err := strconv.ParseInt(rest[tsSpace+1:], 10, 64) + if err != nil { + return signature, fmt.Errorf("invalid timestamp in signature %q", value) + } + identity := rest[:tsSpace] + + if lt := strings.LastIndexByte(identity, '<'); lt >= 0 { + if gt := strings.LastIndexByte(identity[lt:], '>'); gt >= 0 { + signature.Email = identity[lt+1 : lt+gt] + signature.Name = strings.TrimSpace(identity[:lt]) + } else { + signature.Name = strings.TrimSpace(identity) + } + } else { + signature.Name = strings.TrimSpace(identity) + } + + signature.When = time.Unix(seconds, 0).In(parseTimezone(timezone)) + return signature, nil +} + +// parseTimezone converts a git timezone such as "+0800" into a fixed zone so +// the original offset of the commit date is preserved. +func parseTimezone(timezone string) *time.Location { + if len(timezone) != 5 || (timezone[0] != '+' && timezone[0] != '-') { + return time.UTC + } + hours, errH := strconv.Atoi(timezone[1:3]) + minutes, errM := strconv.Atoi(timezone[3:5]) + if errH != nil || errM != nil { + return time.UTC + } + offset := hours*3600 + minutes*60 + if timezone[0] == '-' { + offset = -offset + } + return time.FixedZone(timezone, offset) +} + +// readCatFileObject reads one object from `git cat-file --batch` output. +// It returns io.EOF when the stream is exhausted. +func readCatFileObject(reader *bufio.Reader) (string, []byte, error) { + header, err := reader.ReadString('\n') + if err == io.EOF && header == "" { + return "", nil, io.EOF + } + if err != nil && err != io.EOF { + return "", nil, err + } + + fields := strings.Fields(header) + if len(fields) != 3 { + return "", nil, fmt.Errorf("unexpected cat-file header %q", strings.TrimSpace(header)) + } + if fields[1] != "commit" { + return "", nil, fmt.Errorf("object %s is a %s, not a commit", fields[0], fields[1]) + } + size, err := strconv.Atoi(fields[2]) + if err != nil { + return "", nil, fmt.Errorf("unexpected cat-file size in %q", strings.TrimSpace(header)) + } + + content := make([]byte, size) + if _, err := io.ReadFull(reader, content); err != nil { + return "", nil, err + } + if _, err := reader.Discard(1); err != nil { // trailing newline + return "", nil, err + } + return fields[0], content, nil +} diff --git a/backend/plugins/gitextractor/parser/commitparse_test.go b/backend/plugins/gitextractor/parser/commitparse_test.go new file mode 100644 index 00000000000..fca02f31fb8 --- /dev/null +++ b/backend/plugins/gitextractor/parser/commitparse_test.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const rawCommitObject = `tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 +parent 1111111111111111111111111111111111111111 +parent 2222222222222222222222222222222222222222 +author Jane Doe 1704067200 +0800 +committer DevLake Test 1704070800 -0530 +gpgsig -----BEGIN PGP SIGNATURE----- + fakesignature-line + fakesignature + -----END PGP SIGNATURE----- + +subject line + +body line 1 +body line 2 +` + +func TestParseCommitObject(t *testing.T) { + commit, err := parseCommitObject("abc", []byte(rawCommitObject)) + require.NoError(t, err) + + assert.Equal(t, "abc", commit.Sha) + assert.Equal(t, []string{ + "1111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222", + }, commit.Parents) + + assert.Equal(t, "Jane Doe", commit.Author.Name) + assert.Equal(t, "jane@example.com", commit.Author.Email) + assert.Equal(t, int64(1704067200), commit.Author.When.Unix()) + _, offset := commit.Author.When.Zone() + assert.Equal(t, 8*3600, offset) + + assert.Equal(t, "DevLake Test", commit.Committer.Name) + assert.Equal(t, "test@devlake.apache.org", commit.Committer.Email) + _, offset = commit.Committer.When.Zone() + assert.Equal(t, -(5*3600 + 30*60), offset) + + assert.Equal(t, "subject line\n\nbody line 1\nbody line 2\n", commit.Message) +} + +func TestParseCommitObjectWithoutMessage(t *testing.T) { + content := "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + + "author A 1704067200 +0000\n" + + "committer A 1704067200 +0000\n" + + commit, err := parseCommitObject("abc", []byte(content)) + require.NoError(t, err) + assert.Empty(t, commit.Parents) + assert.Empty(t, commit.Message) + assert.Equal(t, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), commit.Author.When.UTC()) +} + +func TestParseSignature(t *testing.T) { + signature, err := parseSignature("A U Thor 1704067200 +0000") + require.NoError(t, err) + assert.Equal(t, "A U Thor", signature.Name) + assert.Equal(t, "a@example.com", signature.Email) + + // a name containing '<' still resolves to the last email-like part + signature, err = parseSignature("Weird 1704067200 +0000") + require.NoError(t, err) + assert.Equal(t, "Weird ", signature.Name) + assert.Equal(t, "weird@example.com", signature.Email) + + _, err = parseSignature("no timestamp") + assert.Error(t, err) +} + +func TestReadCatFileObject(t *testing.T) { + stream := "abc commit 5\nhello\n" + "def commit 0\n\n" + reader := bufio.NewReader(strings.NewReader(stream)) + + sha, content, err := readCatFileObject(reader) + require.NoError(t, err) + assert.Equal(t, "abc", sha) + assert.Equal(t, "hello", string(content)) + + sha, content, err = readCatFileObject(reader) + require.NoError(t, err) + assert.Equal(t, "def", sha) + assert.Empty(t, content) + + _, _, err = readCatFileObject(reader) + assert.ErrorIs(t, err, io.EOF) +} diff --git a/backend/plugins/gitextractor/parser/gitrunner.go b/backend/plugins/gitextractor/parser/gitrunner.go new file mode 100644 index 00000000000..4d547f8a570 --- /dev/null +++ b/backend/plugins/gitextractor/parser/gitrunner.go @@ -0,0 +1,143 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strings" +) + +// gitRunner runs read-only git commands against one local repository. +// +// Output and Stream both run the git command in a way that never writes to the +// repository and never needs a working tree, so bare clones are supported. +type gitRunner struct { + dir string +} + +func newGitRunner(dir string) *gitRunner { + return &gitRunner{dir: dir} +} + +// Output runs git and returns stdout. Stderr is reported in the error. +func (r *gitRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + var stdout, stderr bytes.Buffer + cmd := r.command(ctx, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + } + return stdout.Bytes(), nil +} + +// OutputLines runs git and returns its stdout lines with the trailing newline +// removed from each line. +func (r *gitRunner) OutputLines(ctx context.Context, args ...string) ([]string, error) { + out, err := r.Output(ctx, args...) + if err != nil { + return nil, err + } + return splitLines(out), nil +} + +// Stream runs git and returns its stdout. The caller must read to EOF and then +// call the returned wait function; closing the reader early kills git. +func (r *gitRunner) Stream(ctx context.Context, args ...string) (io.ReadCloser, func() error, error) { + cmd := r.command(ctx, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, nil, err + } + return stdout, func() error { + err := cmd.Wait() + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + }, nil +} + +// Pipe runs git with stdin and returns a reader over its stdout. +func (r *gitRunner) Pipe(ctx context.Context, stdin []byte, args ...string) (io.ReadCloser, func() error, error) { + cmd := r.command(ctx, args...) + cmd.Stdin = bytes.NewReader(stdin) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, nil, err + } + return stdout, func() error { + err := cmd.Wait() + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + }, nil +} + +func (r *gitRunner) command(ctx context.Context, args ...string) *exec.Cmd { + fullArgs := append([]string{"-c", "core.quotePath=false", "-c", "gc.auto=0", "-c", "maintenance.auto=false"}, args...) + cmd := exec.CommandContext(ctx, "git", fullArgs...) + cmd.Dir = r.dir + cmd.Env = append(cmd.Environ(), + "GIT_OPTIONAL_LOCKS=0", + "GIT_NO_LAZY_FETCH=1", + "GIT_PAGER=cat", + "GIT_TERMINAL_PROMPT=0", + "LC_ALL=C", + ) + return cmd +} + +func errorMessage(stderr []byte, err error) string { + if msg := strings.TrimSpace(string(stderr)); msg != "" { + return msg + } + return err.Error() +} + +func splitLines(out []byte) []string { + text := strings.TrimSuffix(string(out), "\n") + if text == "" { + return nil + } + return strings.Split(text, "\n") +} diff --git a/backend/plugins/gitextractor/parser/numstatparse.go b/backend/plugins/gitextractor/parser/numstatparse.go new file mode 100644 index 00000000000..f7d3fc21ec3 --- /dev/null +++ b/backend/plugins/gitextractor/parser/numstatparse.go @@ -0,0 +1,130 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +const numstatRecordMarker = '\x01' + +type commitFileStat struct { + Path string + Additions int + Deletions int +} + +type commitStats struct { + Sha string + Files []commitFileStat +} + +// parseNumstatStream parses the output of +// +// git log --format=%x01%H --numstat --no-renames -z ... +// +// Each commit is introduced by a \x01 segment; every following NUL +// terminated segment is "additionsdeletionspath". Paths may contain +// tabs, spaces and newlines, but never NUL, so splitting on NUL is safe. +func parseNumstatStream(reader io.Reader, fn func(*commitStats) error) error { + buffered := bufio.NewReaderSize(reader, 256*1024) + + var current *commitStats + flush := func() error { + if current == nil { + return nil + } + if err := fn(current); err != nil { + return err + } + current = nil + return nil + } + + for { + segment, err := buffered.ReadString(0) + if err != nil && err != io.EOF { + return err + } + // ReadString returns data without the delimiter on EOF + segment = strings.TrimSuffix(segment, "\x00") + + switch { + case segment == "": + // nothing to do; covers a trailing NUL + case segment[0] == numstatRecordMarker: + if err := flush(); err != nil { + return err + } + current = &commitStats{Sha: segment[1:]} + default: + if current == nil { + return fmt.Errorf("numstat entry %q before any commit", segment) + } + entry := strings.TrimPrefix(segment, "\n") + stat, err := parseNumstatEntry(entry) + if err != nil { + return fmt.Errorf("commit %s: %w", current.Sha, err) + } + current.Files = append(current.Files, stat) + } + + if err == io.EOF { + break + } + } + return flush() +} + +func parseNumstatEntry(entry string) (commitFileStat, error) { + var stat commitFileStat + parts := strings.SplitN(entry, "\t", 3) + if len(parts) != 3 { + return stat, fmt.Errorf("unexpected numstat entry %q", entry) + } + stat.Path = parts[2] + + additions, err := parseNumstatCount(parts[0]) + if err != nil { + return stat, err + } + deletions, err := parseNumstatCount(parts[1]) + if err != nil { + return stat, err + } + stat.Additions = additions + stat.Deletions = deletions + return stat, nil +} + +// parseNumstatCount parses a numstat count. Binary files are reported as "-" +// and count as zero changed lines. +func parseNumstatCount(value string) (int, error) { + if value == "-" { + return 0, nil + } + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("unexpected numstat count %q", value) + } + return count, nil +} diff --git a/backend/plugins/gitextractor/parser/parse_test.go b/backend/plugins/gitextractor/parser/parse_test.go new file mode 100644 index 00000000000..988a4b87d97 --- /dev/null +++ b/backend/plugins/gitextractor/parser/parse_test.go @@ -0,0 +1,253 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseNumstatStream(t *testing.T) { + stream := strings.Join([]string{ + "\x01sha1\x00", + "\n1\t0\ta.txt\x00", + "-\t-\tbin.dat\x00", + "\x01sha2\x00", + "\x01sha3\x00", + "\n2\t1\tdir with space/f.txt\x00", + "1\t0\tüñï.txt\x00", + "0\t0\tempty.txt\x00", + "1\t0\tnew\nline.txt\x00", + }, "") + + var commits []*commitStats + err := parseNumstatStream(strings.NewReader(stream), func(commit *commitStats) error { + commits = append(commits, commit) + return nil + }) + require.NoError(t, err) + + require.Len(t, commits, 3) + assert.Equal(t, "sha1", commits[0].Sha) + assert.Equal(t, []commitFileStat{ + {Path: "a.txt", Additions: 1}, + {Path: "bin.dat"}, + }, commits[0].Files) + assert.Equal(t, "sha2", commits[1].Sha) + assert.Empty(t, commits[1].Files) + assert.Equal(t, []commitFileStat{ + {Path: "dir with space/f.txt", Additions: 2, Deletions: 1}, + {Path: "üñï.txt", Additions: 1}, + {Path: "empty.txt"}, + {Path: "new\nline.txt", Additions: 1}, + }, commits[2].Files) +} + +func TestParseNumstatStreamRejectsBadInput(t *testing.T) { + err := parseNumstatStream(strings.NewReader("1\t0\ta.txt\x00"), func(*commitStats) error { return nil }) + assert.Error(t, err, "an entry without a commit marker is invalid") + + err = parseNumstatStream(strings.NewReader("\x01sha\x00\nx\t0\ta.txt\x00"), func(*commitStats) error { return nil }) + assert.Error(t, err, "a non-numeric count is invalid") +} + +func TestParsePatchStream(t *testing.T) { + patch := strings.Join([]string{ + "\x01c1", + "diff --git a/a.txt b/a.txt", + "new file mode 100644", + "index 0000000..1111111", + "--- /dev/null", + "+++ b/a.txt", + "@@ -0,0 +1,3 @@", + "+l1", + "+l2", + "+l3", + "\x01c2", + "diff --git a/a.txt b/a.txt", + "index 1111111..2222222 100644", + "--- a/a.txt", + "+++ b/a.txt", + "@@ -1,3 +1,4 @@ heading", + " l1", + "-l2", + "+L2", + " l3", + "+l4", + "\\ No newline at end of file", + "diff --git a/bin.dat b/bin.dat", + "index 3333333..4444444 100644", + "Binary files a/bin.dat and b/bin.dat differ", + "diff --git a/empty.txt b/empty.txt", + "deleted file mode 100644", + "index 5555555..0000000", + "--- a/empty.txt", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-x", + "diff --git a/mode.txt b/mode.txt", + "old mode 100644", + "new mode 100755", + "diff --git \"a/tab\\tname.txt\" \"b/tab\\tname.txt\"", + "index 6666666..7777777 100644", + "--- \"a/tab\\tname.txt\"", + "+++ \"b/tab\\tname.txt\"", + "@@ -1 +1 @@", + "-a", + "+b", + "", + }, "\n") + + var commits []*commitPatch + err := parsePatchStream(strings.NewReader(patch), func(commit *commitPatch) error { + commits = append(commits, commit) + return nil + }) + require.NoError(t, err) + + require.Len(t, commits, 2) + + c1 := commits[0] + assert.Equal(t, "c1", c1.Sha) + require.Len(t, c1.Files, 1) + assert.Equal(t, patchFile{ + OldPath: "a.txt", + NewPath: "a.txt", + Lines: []patchLine{ + {Origin: '+', OldLine: -1, NewLine: 1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 3, HunkNum: 1}, + }, + }, c1.Files[0]) + + c2 := commits[1] + assert.Equal(t, "c2", c2.Sha) + require.Len(t, c2.Files, 5) + + assert.Equal(t, patchFile{ + OldPath: "a.txt", + NewPath: "a.txt", + Lines: []patchLine{ + {Origin: ' ', OldLine: 1, NewLine: 1, HunkNum: 1}, + {Origin: '-', OldLine: 2, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: ' ', OldLine: 3, NewLine: 3, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 4, HunkNum: 1}, + }, + }, c2.Files[0]) + + // binary file: both paths are known, but there are no line changes + assert.Equal(t, patchFile{OldPath: "bin.dat", NewPath: "bin.dat"}, c2.Files[1]) + + // deleted file: the path is reported on both sides + assert.Equal(t, patchFile{ + OldPath: "empty.txt", + NewPath: "empty.txt", + Lines: []patchLine{{Origin: '-', OldLine: 1, NewLine: -1, HunkNum: 1}}, + }, c2.Files[2]) + + // mode-only change + assert.Equal(t, patchFile{OldPath: "mode.txt", NewPath: "mode.txt"}, c2.Files[3]) + + // quoted path + assert.Equal(t, patchFile{ + OldPath: "tab\tname.txt", + NewPath: "tab\tname.txt", + Lines: []patchLine{ + {Origin: '-', OldLine: 1, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 1, HunkNum: 1}, + }, + }, c2.Files[4]) +} + +func TestParsePatchStreamMultipleHunksAndZeroStart(t *testing.T) { + patch := strings.Join([]string{ + "\x01c1", + "diff --git a/a.txt b/a.txt", + "--- a/a.txt", + "+++ b/a.txt", + "@@ -2,1 +2,1 @@", + "-old", + "+new", + "@@ -10,0 +11,2 @@", + "+x", + "+y", + "", + }, "\n") + + var commits []*commitPatch + require.NoError(t, parsePatchStream(strings.NewReader(patch), func(commit *commitPatch) error { + commits = append(commits, commit) + return nil + })) + require.Len(t, commits, 1) + assert.Equal(t, []patchLine{ + {Origin: '-', OldLine: 2, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 11, HunkNum: 2}, + {Origin: '+', OldLine: -1, NewLine: 12, HunkNum: 2}, + }, commits[0].Files[0].Lines) +} + +func TestParseGitVersion(t *testing.T) { + for _, tc := range []struct { + output string + want int + ok bool + }{ + {"git version 2.43.0\n", 2*10000 + 43*100, true}, + {"git version 2.31.0", 2*10000 + 31*100, true}, + {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2, true}, + {"git version 2.56.0", 2*10000 + 56*100, true}, + {"git version 2.30.2", 2*10000 + 30*100 + 2, false}, + {"git version 1.9.0", 1*10000 + 9*100, false}, + } { + got, err := parseGitVersion(tc.output) + if assert.NoError(t, err, tc.output) { + assert.Equal(t, tc.want, got, tc.output) + assert.Equal(t, tc.ok, got >= minGitVersion, tc.output) + } + } + + _, err := parseGitVersion("not git") + assert.Error(t, err) +} + +func TestUnquoteGitPath(t *testing.T) { + for _, tc := range []struct { + in, want string + }{ + {`a/plain.txt`, `a/plain.txt`}, + {`"a/tab\tname.txt"`, "a/tab\tname.txt"}, + {`"a/quote\"name.txt"`, `a/quote"name.txt`}, + {`"a/back\\slash"`, `a/back\slash`}, + {`"a/\303\274ber.txt"`, "a/über.txt"}, + {`"a/new\nline.txt"`, "a/new\nline.txt"}, + } { + assert.Equal(t, tc.want, unquoteGitPath(tc.in), tc.in) + } +} + +func TestResolveRepoPath(t *testing.T) { + assert.Equal(t, "/repo/.git/shallow", resolveRepoPath("/repo", ".git/shallow")) + assert.Equal(t, "/repo/shallow", resolveRepoPath("/repo", "shallow")) + assert.Equal(t, "/abs/shallow", resolveRepoPath("/repo", "/abs/shallow")) +} diff --git a/backend/plugins/gitextractor/parser/patchparse.go b/backend/plugins/gitextractor/parser/patchparse.go new file mode 100644 index 00000000000..3aa27a70f3b --- /dev/null +++ b/backend/plugins/gitextractor/parser/patchparse.go @@ -0,0 +1,332 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +const ( + patchChangeContext = ' ' + patchChangeAddition = '+' + patchChangeDeletion = '-' + + // patchRecordMarker introduces the commit sha in the `git log --format` + // output. A patch line can never start with it: every patch content line + // starts with ' ', '+' or '-'. + patchRecordMarker = "\x01" +) + +type patchLine struct { + Origin byte + OldLine int + NewLine int + HunkNum int +} + +type patchFile struct { + OldPath string + NewPath string + Lines []patchLine +} + +type commitPatch struct { + Sha string + Files []patchFile +} + +type patchParser struct { + commit *commitPatch + current *patchFile + + oldLine int + newLine int + hunkNum int + + inHunk bool +} + +// parsePatchStream parses `git log --format=%x01%H -p ...` output. +func parsePatchStream(reader io.Reader, fn func(*commitPatch) error) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 256*1024), 16*1024*1024) + + var parser *patchParser + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, patchRecordMarker) { + if err := flushPatch(parser, fn); err != nil { + return err + } + parser = &patchParser{commit: &commitPatch{Sha: line[len(patchRecordMarker):]}} + continue + } + if parser == nil { + continue + } + if err := parser.line(line); err != nil { + return err + } + } + if err := scanner.Err(); err != nil { + return err + } + return flushPatch(parser, fn) +} + +func flushPatch(parser *patchParser, fn func(*commitPatch) error) error { + if parser == nil || parser.commit == nil { + return nil + } + parser.endFile() + commit := parser.commit + parser.commit = nil + return fn(commit) +} + +func (p *patchParser) line(line string) error { + switch { + case strings.HasPrefix(line, "diff --git "): + p.endFile() + oldPath, newPath, err := parseDiffGitPaths(strings.TrimPrefix(line, "diff --git ")) + if err != nil { + return err + } + p.current = &patchFile{OldPath: oldPath, NewPath: newPath} + p.inHunk = false + return nil + case strings.HasPrefix(line, "--- "): + if p.current == nil { + p.current = &patchFile{} + } + p.current.OldPath = parsePatchPath(strings.TrimPrefix(line, "--- ")) + return nil + case strings.HasPrefix(line, "+++ "): + if p.current == nil { + p.current = &patchFile{} + } + p.current.NewPath = parsePatchPath(strings.TrimPrefix(line, "+++ ")) + return nil + case strings.HasPrefix(line, "@@ "): + return p.hunkHeader(line) + } + + if !p.inHunk { + return nil + } + + switch line[0] { + case patchChangeContext: + p.appendLine(patchChangeContext, p.oldLine, p.newLine) + p.oldLine++ + p.newLine++ + case patchChangeAddition: + p.appendLine(patchChangeAddition, -1, p.newLine) + p.newLine++ + case patchChangeDeletion: + p.appendLine(patchChangeDeletion, p.oldLine, -1) + p.oldLine++ + case '\\': + // "\ No newline at end of file" + } + return nil +} + +func (p *patchParser) appendLine(origin byte, oldLine, newLine int) { + if p.current == nil { + return + } + p.current.Lines = append(p.current.Lines, patchLine{ + Origin: origin, + OldLine: oldLine, + NewLine: newLine, + HunkNum: p.hunkNum, + }) +} + +func (p *patchParser) hunkHeader(line string) error { + // @@ -oldStart[,oldCount] +newStart[,newCount] @@ optional heading + end := strings.Index(line[3:], " @@") + if end < 0 { + return fmt.Errorf("unexpected hunk header %q", line) + } + fields := strings.SplitN(line[3:3+end], " ", 2) + if len(fields) != 2 { + return fmt.Errorf("unexpected hunk header %q", line) + } + oldStart, err := parseHunkRange(fields[0]) + if err != nil { + return fmt.Errorf("unexpected hunk header %q: %w", line, err) + } + newStart, err := parseHunkRange(fields[1]) + if err != nil { + return fmt.Errorf("unexpected hunk header %q: %w", line, err) + } + p.oldLine = oldStart + p.newLine = newStart + p.hunkNum++ + p.inHunk = true + return nil +} + +func parseHunkRange(value string) (int, error) { + value = strings.TrimPrefix(value, "-") + value = strings.TrimPrefix(value, "+") + if idx := strings.IndexByte(value, ','); idx >= 0 { + value = value[:idx] + } + start, err := strconv.Atoi(value) + if err != nil { + return 0, err + } + return start, nil +} + +func (p *patchParser) endFile() { + if p.current != nil { + // Report the same path on both sides for additions and deletions, so the + // commit file and line change rows always name the file. + if p.current.OldPath == "" { + p.current.OldPath = p.current.NewPath + } + if p.current.NewPath == "" { + p.current.NewPath = p.current.OldPath + } + p.commit.Files = append(p.commit.Files, *p.current) + p.current = nil + } + p.inHunk = false + p.hunkNum = 0 +} + +// parsePatchPath strips the a/ or b/ prefix and unquotes git's C-style path +// quoting. /dev/null becomes the empty path. +func parsePatchPath(value string) string { + if value == "/dev/null" { + return "" + } + return stripGitPathPrefix(unquoteGitPath(value)) +} + +// parseDiffGitPaths extracts both paths from "a/ b/". +// +// Without rename detection both sides are the same path, so finding the split +// point is unambiguous. Quoted paths are parsed as two quoted tokens. +func parseDiffGitPaths(value string) (string, string, error) { + if strings.HasPrefix(value, `"`) { + first, rest, err := readQuotedGitPath(value) + if err != nil { + return "", "", fmt.Errorf("invalid diff header %q: %w", value, err) + } + rest = strings.TrimPrefix(rest, " ") + second, _, err := readQuotedGitPath(rest) + if err != nil { + return "", "", fmt.Errorf("invalid diff header %q: %w", value, err) + } + return stripGitPathPrefix(first), stripGitPathPrefix(second), nil + } + for i := 2; i+3 <= len(value); i++ { + if value[i:i+3] != " b/" { + continue + } + oldPath := value[2:i] + newPath := value[i+3:] + if oldPath == newPath { + return oldPath, newPath, nil + } + } + return "", "", fmt.Errorf("invalid diff header %q", value) +} + +func stripGitPathPrefix(value string) string { + if len(value) > 2 && (value[0] == 'a' || value[0] == 'b') && value[1] == '/' { + return value[2:] + } + return value +} +func readQuotedGitPath(value string) (string, string, error) { + if !strings.HasPrefix(value, `"`) { + return "", value, fmt.Errorf("expected a quoted path") + } + for i := 1; i < len(value); i++ { + if value[i] == '\\' { + i++ + continue + } + if value[i] == '"' { + return unquoteGitPath(value[:i+1]), value[i+1:], nil + } + } + return "", value, fmt.Errorf("unterminated quoted path") +} + +// unquoteGitPath decodes git's C-style quoted paths, e.g. "a/\303\274ber". +func unquoteGitPath(value string) string { + if len(value) < 2 || value[0] != '"' || value[len(value)-1] != '"' { + return value + } + var out strings.Builder + for i := 1; i < len(value)-1; i++ { + char := value[i] + if char != '\\' { + out.WriteByte(char) + continue + } + i++ + if i >= len(value)-1 { + break + } + switch value[i] { + case 'a': + out.WriteByte('\a') + case 'b': + out.WriteByte('\b') + case 'f': + out.WriteByte('\f') + case 'n': + out.WriteByte('\n') + case 'r': + out.WriteByte('\r') + case 't': + out.WriteByte('\t') + case 'v': + out.WriteByte('\v') + case '\\', '"': + out.WriteByte(value[i]) + case '0', '1', '2', '3', '4', '5', '6', '7': + end := i + 3 + if end > len(value)-1 { + end = len(value) - 1 + } + code, err := strconv.ParseUint(value[i:end], 8, 8) + if err != nil { + out.WriteByte(value[i]) + continue + } + out.WriteByte(byte(code)) + i = end - 1 + default: + out.WriteByte(value[i]) + } + } + return out.String() +} diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go new file mode 100644 index 00000000000..60ef6b70975 --- /dev/null +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -0,0 +1,784 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/log" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/code" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/plugins/gitextractor/models" +) + +// minGitVersion is the first release with --diff-merges=first-parent, which is +// required to diff merge commits against their first parent only. +const minGitVersion = 2*10000 + 31*100 + +// commitBatchSize bounds how many commits are fetched and diffed at a time so +// that a repository with a long history does not need to be held in memory. +const commitBatchSize = 1024 + +var _ RepoCollector = (*GitcliRepoCollector)(nil) + +// GitcliRepoCollector collects git data with the git command line. It replaces +// the previous libgit2 and go-git implementations, so the server no longer +// needs a git library or cgo. +type GitcliRepoCollector struct { + id string + logger log.Logger + store models.Store + git *gitRunner + cleanup func() +} + +func NewGitcliRepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*GitcliRepoCollector, errors.Error) { + git := newGitRunner(localDir) + if _, err := git.Output(context.Background(), "rev-parse", "--git-dir"); err != nil { + return nil, errors.Convert(err) + } + version, err := gitVersion(git) + if err != nil { + return nil, errors.Convert(err) + } + if version < minGitVersion { + return nil, errors.Default.New(fmt.Sprintf( + "git %s is not supported, gitextractor requires git 2.31 or newer", formatGitVersion(version))) + } + return &GitcliRepoCollector{ + id: repoId, + logger: logger, + store: store, + git: git, + }, nil +} + +func (r *GitcliRepoCollector) SetCleanUp(f func()) error { + if f != nil { + r.cleanup = f + } + return nil +} + +func (r *GitcliRepoCollector) Close(ctx context.Context) error { + defer func() { + if r.cleanup != nil { + r.cleanup() + } + }() + return r.store.Close() +} + +// CollectAll The main parser subtask +func (r *GitcliRepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { + subtaskCtx.SetProgress(0, -1) + if err := r.CollectTags(subtaskCtx); err != nil { + return err + } + if err := r.CollectBranches(subtaskCtx); err != nil { + return err + } + if err := r.CollectCommits(subtaskCtx); err != nil { + return err + } + if *subtaskCtx.GetData().(*GitExtractorTaskData).Options.SkipCommitStat { + return nil + } + return r.CollectDiffLine(subtaskCtx) +} + +// CountTags Count git tags subtask +func (r *GitcliRepoCollector) CountTags(ctx context.Context) (int, error) { + tags, err := r.tagRefs(ctx) + if err != nil { + return 0, err + } + return len(tags), nil +} + +// CountBranches count the number of branches in a git repo +func (r *GitcliRepoCollector) CountBranches(ctx context.Context) (int, error) { + branches, err := r.branchRefs(ctx) + if err != nil { + return 0, err + } + head, err := r.headRef(ctx) + if err != nil { + return 0, err + } + count := 0 + for _, branch := range branches { + if branch.refname != head { + count++ + } + } + return count, nil +} + +// CountCommits count the number of commits in a git repo +func (r *GitcliRepoCollector) CountCommits(ctx context.Context) (int, error) { + out, err := r.git.Output(ctx, "rev-list", "--all", "--count") + if err != nil { + return 0, err + } + count, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + return 0, fmt.Errorf("unexpected commit count %q", strings.TrimSpace(string(out))) + } + return count, nil +} + +// CollectTags Collect Tags data +func (r *GitcliRepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { + tags, err := r.tagRefs(subtaskCtx.GetContext()) + if err != nil { + return err + } + for _, tag := range tags { + commitSha := tag.peeledSha + if commitSha == "" { + commitSha = tag.sha + } + ref := &code.Ref{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, tag.refname)}, + RepoId: r.id, + Name: tag.refname, + CommitSha: commitSha, + RefType: TAG, + } + if err := r.store.Refs(ref); err != nil { + return err + } + subtaskCtx.IncProgress(1) + } + return nil +} + +// CollectBranches Collect branch data +func (r *GitcliRepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { + branches, err := r.branchRefs(subtaskCtx.GetContext()) + if err != nil { + return err + } + head, err := r.headRef(subtaskCtx.GetContext()) + if err != nil { + return err + } + for _, branch := range branches { + ref := &code.Ref{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, branch.name)}, + RepoId: r.id, + Name: branch.name, + CommitSha: branch.sha, + RefType: BRANCH, + IsDefault: branch.refname == head, + } + if err := r.store.Refs(ref); err != nil { + return err + } + subtaskCtx.IncProgress(1) + } + return nil +} + +// CollectCommits Collect data from each commit, we can also get the diff line +func (r *GitcliRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) { + ctx := subtaskCtx.GetContext() + taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options + + shas, err := r.git.OutputLines(ctx, "rev-list", "--all") + if err != nil { + return err + } + if len(shas) == 0 { + return nil + } + present := make(map[string]struct{}, len(shas)) + for _, sha := range shas { + present[sha] = struct{}{} + } + shallow, err := r.shallowCommits(ctx) + if err != nil { + return err + } + var componentMap map[string]*regexp.Regexp + if !*taskOpts.SkipCommitFiles { + componentMap, err = r.componentMap(subtaskCtx) + if err != nil { + return err + } + } + excluded := normalizeExtensions(taskOpts.ExcludeFileExtensions) + + for start := 0; start < len(shas); start += commitBatchSize { + end := start + commitBatchSize + if end > len(shas) { + end = len(shas) + } + batch := shas[start:end] + + var statsBySha map[string]*commitStats + if !*taskOpts.SkipCommitStat { + statsBySha, err = r.commitStats(ctx, batch) + if err != nil { + return err + } + } + commits, err := r.readCommits(ctx, batch) + if err != nil { + return err + } + for _, metadata := range commits { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // A shallow boundary commit's parent is not in the object + // database. Diffing it would produce a full-tree diff and inflate + // the addition count, so it is skipped entirely. + if _, ok := shallow[metadata.Sha]; ok { + r.logger.Info("skip commit %s because it is a shallow boundary", metadata.Sha) + continue + } + // The same guard as above, for repositories where the boundary is + // not recorded in the shallow file. + if len(metadata.Parents) > 0 { + if _, ok := present[metadata.Parents[0]]; !ok { + r.logger.Info("skip commit %s because it has no parent commit", metadata.Sha) + continue + } + } + + commit := &code.Commit{ + Sha: metadata.Sha, + Message: metadata.Message, + AuthorName: metadata.Author.Name, + AuthorEmail: metadata.Author.Email, + AuthorId: metadata.Author.Email, + AuthoredDate: metadata.Author.When, + CommitterName: metadata.Committer.Name, + CommitterEmail: metadata.Committer.Email, + CommitterId: metadata.Committer.Email, + CommittedDate: metadata.Committer.When, + } + + var files []commitFileStat + if stats := statsBySha[metadata.Sha]; stats != nil { + files = stats.Files + } + for _, file := range files { + if isExcludedPath(file.Path, excluded) { + continue + } + commit.Additions += file.Additions + commit.Deletions += file.Deletions + } + + if err := r.storeParentCommits(metadata, present); err != nil { + return err + } + if err := r.store.Commits(commit); err != nil { + return err + } + if err := r.store.RepoCommits(&code.RepoCommit{RepoId: r.id, CommitSha: commit.Sha}); err != nil { + return err + } + + if !*taskOpts.SkipCommitFiles { + for _, file := range files { + if isExcludedPath(file.Path, excluded) { + continue + } + if err := r.storeCommitFile(commit.Sha, file, componentMap); err != nil { + return err + } + } + } + subtaskCtx.IncProgress(1) + } + } + return nil +} + +func (r *GitcliRepoCollector) storeParentCommits(commit *parsedCommit, present map[string]struct{}) errors.Error { + var parents []*code.CommitParent + for _, parentSha := range commit.Parents { + // Skip parents that are not in the object database, which happens for + // shallow clones, instead of storing a dangling reference. + if _, ok := present[parentSha]; !ok { + continue + } + parents = append(parents, &code.CommitParent{ + CommitSha: commit.Sha, + ParentCommitSha: parentSha, + }) + } + return r.store.CommitParents(parents) +} + +// storeCommitFile stores one changed file of a commit. Component rows are only +// written when the repository has components configured: without them every row +// would be "Default" for every changed file, which is pure write amplification. +func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileStat, componentMap map[string]*regexp.Regexp) errors.Error { + commitFile := &code.CommitFile{ + CommitSha: commitSha, + FilePath: file.Path, + Additions: file.Additions, + Deletions: file.Deletions, + } + commitFile.Id = genCommitFileId(commitFile.CommitSha, commitFile.FilePath) + + if len(componentMap) > 0 { + componentName := "Default" + for _, name := range sortedComponentNames(componentMap) { + if componentMap[name].MatchString(commitFile.FilePath) { + componentName = name + break + } + } + if err := r.store.CommitFileComponents(&code.CommitFileComponent{ + CommitFileId: commitFile.Id, + ComponentName: componentName, + }); err != nil { + return err + } + } + return r.store.CommitFiles(commitFile) +} + +// CollectDiffLine replays the first-parent history of HEAD and records the +// origin commit of every line, plus the per-line change rows. +func (r *GitcliRepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { + ctx := subtaskCtx.GetContext() + + if _, err := r.git.Output(ctx, "rev-parse", "--verify", "HEAD"); err != nil { + // an empty repository has no HEAD and therefore no snapshot + r.logger.Info("skip diff line collection because the repository has no HEAD") + return nil + } + + reader, wait, err := r.git.Stream(ctx, + "log", "--first-parent", "--reverse", "--no-renames", "-p", "-U3", + "--format="+patchRecordMarker+"%H", + "--no-color", "--no-ext-diff", "--no-show-signature", "--no-notes", + "HEAD", + ) + if err != nil { + return err + } + + snapshot := make(map[string]*models.FileBlame) + parseErr := parsePatchStream(reader, func(patch *commitPatch) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return r.storeSnapshotForCommit(patch, snapshot) + }) + waitErr := wait() + if parseErr != nil { + return parseErr + } + if waitErr != nil { + return waitErr + } + + db := subtaskCtx.GetDal() + if err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id)); err != nil { + return err + } + + files := make([]string, 0, len(snapshot)) + for file := range snapshot { + files = append(files, file) + } + sort.Strings(files) + for _, file := range files { + lineNo := 0 + for element := snapshot[file].Lines.Front(); element != nil; element = element.Next() { + lineNo++ + commitSha, ok := element.Value.(string) + if !ok { + // A placeholder without a commit should not happen, but do not + // fail the whole collection if it does. + r.logger.Warn(nil, "empty snapshot line for %s:%d", file, lineNo) + continue + } + if err := r.store.RepoSnapshot(&code.RepoSnapshot{ + RepoId: r.id, + CommitSha: commitSha, + FilePath: file, + LineNo: lineNo, + }); err != nil { + return err + } + } + } + r.logger.Info("collect snapshot finished") + return nil +} + +func (r *GitcliRepoCollector) storeSnapshotForCommit(patch *commitPatch, snapshot map[string]*models.FileBlame) error { + pending := &snapshotChanges{} + lastFile := "" + + for _, file := range patch.Files { + if _, ok := snapshot[file.OldPath]; !ok { + fileBlame, err := models.NewFileBlame() + if err != nil { + return err + } + snapshot[file.OldPath] = fileBlame + } + if lastFile == "" { + lastFile = file.NewPath + } else if lastFile != file.NewPath { + applySnapshotChanges(snapshot, lastFile, patch.Sha, pending) + pending = &snapshotChanges{} + lastFile = file.NewPath + } + + for _, line := range file.Lines { + change := &code.CommitLineChange{ + CommitSha: patch.Sha, + ChangedType: lineChangeType(line.Origin), + LineNoNew: line.NewLine, + LineNoOld: line.OldLine, + OldFilePath: file.OldPath, + NewFilePath: file.NewPath, + HunkNum: line.HunkNum, + Id: fmt.Sprintf("%s:%s:%d:%d", patch.Sha, file.NewPath, line.OldLine, line.NewLine), + } + switch line.Origin { + case patchChangeAddition: + pending.added = append(pending.added, line) + case patchChangeDeletion: + if blame := snapshot[file.OldPath]; blame != nil { + if element := blame.Find(line.OldLine); element != nil && element.Value != nil { + change.PrevCommit = element.Value.(string) + } + } + pending.deleted = append(pending.deleted, line) + } + if err := r.store.CommitLineChange(change); err != nil { + return err + } + } + } + applySnapshotChanges(snapshot, lastFile, patch.Sha, pending) + return nil +} + +type snapshotChanges struct { + deleted []patchLine + added []patchLine +} + +// applySnapshotChanges applies the changes of one file to the line snapshot: +// deletions remove the old line numbers and additions insert the commit sha at +// the new line numbers. +func applySnapshotChanges(snapshot map[string]*models.FileBlame, filePath, commitSha string, changes *snapshotChanges) { + fileBlame, ok := snapshot[filePath] + if !ok || fileBlame == nil || (len(changes.deleted) == 0 && len(changes.added) == 0) { + return + } + sort.SliceStable(changes.deleted, func(i, j int) bool { + return changes.deleted[i].OldLine > changes.deleted[j].OldLine + }) + for _, line := range changes.deleted { + fileBlame.RemoveLine(line.OldLine) + } + for _, line := range changes.added { + fileBlame.AddLine(line.NewLine, commitSha) + } +} + +func lineChangeType(origin byte) string { + switch origin { + case patchChangeAddition: + return "Addition" + case patchChangeDeletion: + return "Deletion" + default: + return "Context" + } +} + +// readCommits returns the commit objects of the batch in the order of the batch. +func (r *GitcliRepoCollector) readCommits(ctx context.Context, shas []string) ([]*parsedCommit, error) { + stdin := []byte(strings.Join(shas, "\n") + "\n") + reader, wait, err := r.git.Pipe(ctx, stdin, "cat-file", "--batch") + if err != nil { + return nil, err + } + buffered := bufio.NewReaderSize(reader, 256*1024) + commits := make([]*parsedCommit, 0, len(shas)) + for { + sha, content, readErr := readCatFileObject(buffered) + if readErr == io.EOF { + break + } + if readErr != nil { + _ = wait() + return nil, readErr + } + commit, parseErr := parseCommitObject(sha, content) + if parseErr != nil { + _ = wait() + return nil, parseErr + } + commits = append(commits, commit) + } + if err := wait(); err != nil { + return nil, err + } + return commits, nil +} + +// commitStats returns the per-file line counts of the batch. +func (r *GitcliRepoCollector) commitStats(ctx context.Context, shas []string) (map[string]*commitStats, error) { + stdin := []byte(strings.Join(shas, "\n") + "\n") + reader, wait, err := r.git.Pipe(ctx, stdin, + "log", "--no-walk=unsorted", "--stdin", "--root", "--diff-merges=first-parent", + "--numstat", "--no-renames", "-z", "--format="+patchRecordMarker+"%H", + "--no-color", "--no-ext-diff", "--no-show-signature", "--no-notes", + ) + if err != nil { + return nil, err + } + statsBySha := make(map[string]*commitStats, len(shas)) + err = parseNumstatStream(reader, func(stats *commitStats) error { + statsBySha[stats.Sha] = stats + return nil + }) + waitErr := wait() + if err != nil { + return nil, err + } + if waitErr != nil { + return nil, waitErr + } + return statsBySha, nil +} + +// shallowCommits returns the commits recorded as shallow boundaries. +func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]struct{}, error) { + shallow := make(map[string]struct{}) + out, err := r.git.Output(ctx, "rev-parse", "--git-path", "shallow") + if err != nil { + return shallow, err + } + content, err := os.ReadFile(resolveRepoPath(r.git.dir, strings.TrimSpace(string(out)))) + if os.IsNotExist(err) { + return shallow, nil + } + if err != nil { + return shallow, err + } + for _, line := range strings.Split(string(content), "\n") { + if sha := strings.TrimSpace(line); sha != "" { + shallow[sha] = struct{}{} + } + } + return shallow, nil +} + +// resolveRepoPath turns a git path that is relative to the repository into an +// absolute one. `git rev-parse --git-path` returns paths relative to the +// repository root, but the collector's working directory is not the repository. +func resolveRepoPath(repoDir, gitPath string) string { + if filepath.IsAbs(gitPath) { + return gitPath + } + return filepath.Join(repoDir, gitPath) +} + +type tagRef struct { + refname string + sha string + peeledSha string +} + +func (r *GitcliRepoCollector) tagRefs(ctx context.Context) ([]tagRef, error) { + lines, err := r.git.OutputLines(ctx, "for-each-ref", + "--format=%(refname)%00%(objectname)%00%(*objectname)", + "refs/tags") + if err != nil { + return nil, err + } + tags := make([]tagRef, 0, len(lines)) + for _, line := range lines { + fields := strings.SplitN(line, "\x00", 3) + if len(fields) != 3 { + return nil, fmt.Errorf("unexpected tag ref %q", line) + } + tags = append(tags, tagRef{refname: fields[0], sha: fields[1], peeledSha: fields[2]}) + } + return tags, nil +} + +type branchRef struct { + refname string + name string + sha string +} + +func (r *GitcliRepoCollector) branchRefs(ctx context.Context) ([]branchRef, error) { + lines, err := r.git.OutputLines(ctx, "for-each-ref", + "--format=%(refname)%00%(objectname)", + "refs/heads", "refs/remotes") + if err != nil { + return nil, err + } + branches := make([]branchRef, 0, len(lines)) + for _, line := range lines { + fields := strings.SplitN(line, "\x00", 2) + if len(fields) != 2 { + return nil, fmt.Errorf("unexpected branch ref %q", line) + } + name := strings.TrimPrefix(fields[0], "refs/heads/") + name = strings.TrimPrefix(name, "refs/remotes/") + branches = append(branches, branchRef{refname: fields[0], name: name, sha: fields[1]}) + } + return branches, nil +} + +func (r *GitcliRepoCollector) headRef(ctx context.Context) (string, error) { + out, err := r.git.Output(ctx, "symbolic-ref", "-q", "HEAD") + if err != nil { + // a detached HEAD has no default branch + return "", nil + } + return strings.TrimSpace(string(out)), nil +} + +func (r *GitcliRepoCollector) componentMap(subtaskCtx plugin.SubTaskContext) (map[string]*regexp.Regexp, error) { + db := subtaskCtx.GetDal() + components := make([]code.Component, 0) + if err := db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)); err != nil { + return nil, err + } + componentMap := make(map[string]*regexp.Regexp, len(components)) + for _, component := range components { + componentMap[component.Name] = regexp.MustCompile(component.PathRegex) + } + return componentMap, nil +} + +func sortedComponentNames(componentMap map[string]*regexp.Regexp) []string { + names := make([]string, 0, len(componentMap)) + for name := range componentMap { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// genCommitFileId builds the commit file primary key. With some long paths the +// varchar(255) columns were not enough for both the id and the file path, so +// the path is compressed with sha256. +func genCommitFileId(commitSha, filePath string) string { + shaFilePath := sha256.New() + shaFilePath.Write([]byte(filePath)) + return commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) +} + +func normalizeExtensions(extensions []string) map[string]struct{} { + excluded := make(map[string]struct{}, len(extensions)) + for _, extension := range extensions { + value := strings.ToLower(strings.TrimSpace(extension)) + if value == "" { + continue + } + excluded[value] = struct{}{} + } + return excluded +} + +func isExcludedPath(path string, excluded map[string]struct{}) bool { + if len(excluded) == 0 { + return false + } + lower := strings.ToLower(path) + for extension := range excluded { + if strings.HasSuffix(lower, extension) { + return true + } + } + return false +} + +// gitVersion returns the version of git as major*10000 + minor*100 + patch. +func gitVersion(git *gitRunner) (int, error) { + out, err := git.Output(context.Background(), "version") + if err != nil { + return 0, err + } + return parseGitVersion(string(out)) +} + +func parseGitVersion(output string) (int, error) { + // e.g. "git version 2.43.0", "git version 2.39.2.windows.1" + fields := strings.Fields(output) + if len(fields) < 3 || fields[0] != "git" || fields[1] != "version" { + return 0, fmt.Errorf("unexpected git version output %q", strings.TrimSpace(output)) + } + parts := strings.SplitN(fields[2], ".", 3) + if len(parts) < 2 { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + major, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + patch := 0 + if len(parts) == 3 { + if idx := strings.IndexFunc(parts[2], func(r rune) bool { return r < '0' || r > '9' }); idx >= 0 { + parts[2] = parts[2][:idx] + } + if parts[2] != "" { + patch, _ = strconv.Atoi(parts[2]) + } + } + return major*10000 + minor*100 + patch, nil +} + +func formatGitVersion(version int) string { + return fmt.Sprintf("%d.%d.%d", version/10000, version%10000/100, version%100) +} diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go b/backend/plugins/gitextractor/parser/repo_gogit.go deleted file mode 100644 index fe254e85935..00000000000 --- a/backend/plugins/gitextractor/parser/repo_gogit.go +++ /dev/null @@ -1,626 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "regexp" - "strings" - - "github.com/apache/devlake/core/dal" - "github.com/apache/devlake/core/errors" - "github.com/apache/devlake/core/log" - "github.com/apache/devlake/core/models/domainlayer" - "github.com/apache/devlake/core/models/domainlayer/code" - "github.com/apache/devlake/core/plugin" - "github.com/apache/devlake/plugins/gitextractor/models" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -type GogitRepoCollector struct { - id string - logger log.Logger - store models.Store - repo *gogit.Repository - cleanUp func() -} - -func NewGogitRepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*GogitRepoCollector, errors.Error) { - repo, err := gogit.PlainOpen(localDir) - if err != nil { - return nil, errors.Convert(err) - } - return &GogitRepoCollector{ - id: repoId, - logger: logger, - store: store, - repo: repo, - }, nil -} - -func (r *GogitRepoCollector) SetCleanUp(f func()) error { - if f != nil { - r.cleanUp = f - } - return nil -} - -func (r *GogitRepoCollector) Close(ctx context.Context) error { - if err := r.store.Close(); err != nil { - return err - } - if r.cleanUp != nil { - r.cleanUp() - } - return nil -} - -// CollectAll The main parser subtask -func (r *GogitRepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { - subtaskCtx.SetProgress(0, -1) - err := r.CollectTags(subtaskCtx) - if err != nil { - return err - } - err = r.CollectBranches(subtaskCtx) - if err != nil { - return err - } - err = r.CollectCommits(subtaskCtx) - if err != nil { - return err - } - return r.CollectDiffLine(subtaskCtx) -} - -// CountTags Count git tags subtask -func (r *GogitRepoCollector) CountTags(ctx context.Context) (int, error) { - iter, err := r.repo.Tags() - if err != nil { - return 0, err - } - var tagsCount int - if err := iter.ForEach(func(reference *plumbing.Reference) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - tagsCount += 1 - return nil - }); err != nil { - return 0, err - } - return tagsCount, nil -} - -// CountBranches count the number of branches in a git repo -func (r *GogitRepoCollector) CountBranches(ctx context.Context) (int, error) { - refIter, err := r.repo.Storer.IterReferences() - if err != nil { - return 0, err - } - branchIter := storer.NewReferenceFilteredIter( - func(r *plumbing.Reference) bool { - return r.Name().IsBranch() || r.Name().IsRemote() - }, refIter) - var branchesCount int - - headRef, err := r.repo.Head() - if err != nil { - return 0, err - } - if err := branchIter.ForEach(func(reference *plumbing.Reference) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if reference.Name() != headRef.Name() { - branchesCount += 1 - } - return nil - }); err != nil { - return 0, err - } - return branchesCount, nil -} - -// CountCommits count the number of commits in a git repo -func (r *GogitRepoCollector) CountCommits(ctx context.Context) (int, error) { - iter, err := r.repo.CommitObjects() - if err != nil { - return 0, err - } - var count int - if err := iter.ForEach(func(commit *object.Commit) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - count += 1 - return nil - }); err != nil { - return 0, err - } - return count, nil -} - -// CollectTags Collect Tags data -func (r *GogitRepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { - tagIter, err := r.repo.Tags() - if err != nil { - return err - } - if err := tagIter.ForEach(func(ref *plumbing.Reference) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - tagCommit := ref.Hash().String() - _, err := r.repo.CommitObject(ref.Hash()) - if err != nil && errors.Is(err, plumbing.ErrObjectNotFound) { - h, err := r.repo.ResolveRevision(plumbing.Revision(ref.Name())) - if err != nil { - return err - } - tagCommit = h.String() - } - name := ref.Name().String() - if tagCommit != "" { - codeRef := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: tagCommit, - RefType: TAG, - } - err = r.store.Refs(codeRef) - if err != nil { - return err - } - subtaskCtx.IncProgress(1) - } - return nil - }); err != nil { - return err - } - return nil -} - -// CollectBranches Collect branch data -func (r *GogitRepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { - refIter, err := r.repo.Storer.IterReferences() - if err != nil { - return err - } - branchIter := storer.NewReferenceFilteredIter( - func(r *plumbing.Reference) bool { - return r.Name().IsBranch() || r.Name().IsRemote() - }, refIter) - headRef, err := r.repo.Head() - if err != nil { - return err - } - if err := branchIter.ForEach(func(ref *plumbing.Reference) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - name := ref.Name().Short() - sha := ref.Hash().String() - _, err := r.repo.CommitObject(ref.Hash()) - if err != nil && errors.Is(err, plumbing.ErrObjectNotFound) { - // handle commit sha like "0000000000000000000000000000000000000000" - h, err := r.repo.ResolveRevision(plumbing.Revision(ref.Name())) - if err != nil { - return err - } - sha = h.String() - } - codeRef := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: sha, - RefType: BRANCH, - IsDefault: ref.Name() == headRef.Name(), - } - if err := r.store.Refs(codeRef); err != nil { - return err - } - subtaskCtx.IncProgress(1) - return nil - }); err != nil { - return err - } - return nil -} - -func (r *GogitRepoCollector) getComponentMap(subtaskCtx plugin.SubTaskContext) (map[string]*regexp.Regexp, error) { - db := subtaskCtx.GetDal() - components := make([]code.Component, 0) - err := db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) - if err != nil { - return nil, err - } - componentMap := make(map[string]*regexp.Regexp) - for _, component := range components { - componentMap[component.Name] = regexp.MustCompile(component.PathRegex) - } - return componentMap, nil -} - -// CollectCommits Collect data from each commit, we can also get the diff line -func (r *GogitRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) { - taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options - // check it first - componentMap, err := r.getComponentMap(subtaskCtx) - if err != nil { - return err - } - - repo := r.repo - store := r.store - - commitsObjectsIter, err := repo.CommitObjects() - if err != nil { - return err - } - - if err := commitsObjectsIter.ForEach(func(commit *object.Commit) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - commitSha := commit.Hash.String() - - if commit.NumParents() != 0 { - _, err := commit.Parents().Next() - if err != nil { - if err == plumbing.ErrObjectNotFound { - // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. - // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting - // the correct addition/deletion data in the database with an absurdly large addition number. - r.logger.Info("skip commit %s because it has no parent commit", commitSha) - return nil - } - return err - } - } - codeCommit := &code.Commit{ - Sha: commitSha, - Message: commit.Message, - AuthorName: commit.Author.Name, - AuthorEmail: commit.Author.Email, - AuthorId: commit.Author.Email, - AuthoredDate: commit.Author.When, - CommitterName: commit.Committer.Name, - CommitterEmail: commit.Committer.Email, - CommitterId: commit.Committer.Email, - CommittedDate: commit.Committer.When, - } - if err = r.storeParentCommits(commitSha, commit); err != nil { - return err - } - - if !*taskOpts.SkipCommitStat { - stats, err := commit.StatsContext(subtaskCtx.GetContext()) - if err != nil { - return err - } else { - excluded := map[string]struct{}{} - for _, ext := range taskOpts.ExcludeFileExtensions { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - for _, stat := range stats { - nameLower := strings.ToLower(stat.Name) - skip := false - for ext := range excluded { - if strings.HasSuffix(nameLower, ext) { - skip = true - break - } - } - if skip { - continue - } - codeCommit.Additions += stat.Addition - // In some repos, deletion may be zero, which is different from git log --stat. - // It seems go-git doesn't get the correct changes. - // I have run object.DiffTreeWithOptions manually with different diff algorithms, - // but get the same result with StatsContext. - // I cannot reproduce it with another repo. - // A similar issue: https://github.com/go-git/go-git/issues/367 - codeCommit.Deletions += stat.Deletion - } - } - } - - err = store.Commits(codeCommit) - if err != nil { - return err - } - - codeRepoCommit := &code.RepoCommit{ - RepoId: r.id, - CommitSha: commitSha, - } - err = store.RepoCommits(codeRepoCommit) - if err != nil { - return err - } - if !*taskOpts.SkipCommitFiles { - if err := r.storeDiffCommitFilesComparedToParent(subtaskCtx, componentMap, commit, taskOpts.ExcludeFileExtensions); err != nil { - return err - } - } - subtaskCtx.IncProgress(1) - return nil - }); err != nil { - return err - } - return -} - -func (r *GogitRepoCollector) storeParentCommits(commitSha string, commit *object.Commit) error { - if commit == nil { - return nil - } - var commitParents []*code.CommitParent - for i := 0; i < commit.NumParents(); i++ { - parent, err := commit.Parent(i) - if err != nil { - // parent commit might not exist when repo is shallow cloned (tradeoff of supporting timeAfter parameter) - if err.Error() == "object not found" { - continue - } - return err - } - if parent != nil { - if parentCommitSha := parent.Hash.String(); parentCommitSha != "" { - commitParents = append(commitParents, &code.CommitParent{ - CommitSha: commitSha, - ParentCommitSha: parentCommitSha, - }) - } - } - } - return r.store.CommitParents(commitParents) -} - -func (r *GogitRepoCollector) getCurrentAndParentTree(ctx context.Context, commit *object.Commit) (*object.Tree, *object.Tree, error) { - if _, err := commit.Stats(); err != nil { - return nil, nil, err - } - commitTree, err := commit.Tree() - if err != nil { - return nil, nil, err - } - var firstParentTree *object.Tree - if commit.NumParents() > 0 { - firstParent, err := commit.Parents().Next() - if err != nil { - return nil, nil, err - } - firstParentTree, err = firstParent.Tree() - if err != nil { - return nil, nil, err - } - } - return commitTree, firstParentTree, nil -} - -func (r *GogitRepoCollector) storeDiffCommitFilesComparedToParent(subtaskCtx plugin.SubTaskContext, componentMap map[string]*regexp.Regexp, commit *object.Commit, excludeExts []string) (err error) { - commitTree, firstParentTree, err := r.getCurrentAndParentTree(subtaskCtx.GetContext(), commit) - if err != nil { - return err - } - // no parent, doesn't need to patch - patch, err := firstParentTree.PatchContext(subtaskCtx.GetContext(), commitTree) - if err != nil { - return err - } - // normalize exclusions - excluded := map[string]struct{}{} - for _, ext := range excludeExts { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - for _, p := range patch.Stats() { - commitFile := &code.CommitFile{ - CommitSha: commit.Hash.String(), - } - fileName := p.Name - commitFile.FilePath = fileName - if len(excluded) > 0 { - lower := strings.ToLower(fileName) - skip := false - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - skip = true - break - } - } - if skip { - continue - } - } - commitFile.Id = genCommitFileId(commitFile.CommitSha, fileName) - commitFile.Deletions = p.Deletion - commitFile.Additions = p.Addition - if err := r.storeCommitFileComponents(subtaskCtx, componentMap, commitFile.Id, commitFile.FilePath); err != nil { - return err - } - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - return nil - } - } - return nil -} - -// With some long path,the varchar(255) was not enough both ID and file_path -// So we use the hash to compress the path in ID and add length of file_path. -// Use commitSha and the sha256 of FilePath to create id -func genCommitFileId(commitSha, filePath string) string { - shaFilePath := sha256.New() - shaFilePath.Write([]byte(filePath)) - return commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) -} - -func (r *GogitRepoCollector) storeCommitFileComponents(subtaskCtx plugin.SubTaskContext, componentMap map[string]*regexp.Regexp, commitFileId string, commitFilePath string) error { - if commitFileId == "" || commitFilePath == "" { - return errors.Default.New("commit id r commit file path is empty") - } - commitFileComponent := &code.CommitFileComponent{ - CommitFileId: commitFileId, - ComponentName: "Default", - } - for component, reg := range componentMap { - if reg.MatchString(commitFilePath) { - commitFileComponent.ComponentName = component - break - } - } - return r.store.CommitFileComponents(commitFileComponent) -} - -// storeRepoSnapshot depends on commit list's order. -func (r *GogitRepoCollector) storeRepoSnapshot(subtaskCtx plugin.SubTaskContext, commitList []*object.Commit) error { - ctx := subtaskCtx.GetContext() - snapshot := make(map[string][]string) // {"filePathAndName": ["line1 commit sha", "line2 commit sha"]} - for _, commit := range commitList { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - commitTree, firstParentTree, err := r.getCurrentAndParentTree(ctx, commit) - if err != nil { - return err - } - patch, err := firstParentTree.PatchContext(subtaskCtx.GetContext(), commitTree) - if err != nil { - return err - } - for _, p := range patch.Stats() { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - fileName := p.Name - if _, ok := snapshot[fileName]; !ok { - snapshot[fileName] = []string{} - } - blameResults, err := gogit.Blame(commit, fileName) - if err != nil { - return err - } - var newBlames []string - for _, blameResult := range blameResults.Lines { - newBlames = append(newBlames, blameResult.Hash.String()) - } - snapshot[fileName] = newBlames - } - } - // store snapshots - for fileName, lineBlames := range snapshot { - for idx, lineBlameHash := range lineBlames { - lineNo := idx + 1 - repoSnapshot := &code.RepoSnapshot{ - RepoId: r.id, - CommitSha: lineBlameHash, - FilePath: fileName, - LineNo: lineNo, - } - if err := r.store.RepoSnapshot(repoSnapshot); err != nil { - r.logger.Error(err, "store RepoSnapshot error") - return err - } - } - } - return nil -} - -func (r *GogitRepoCollector) GetCommitList(subtaskCtx plugin.SubTaskContext) ([]*object.Commit, error) { - var commitList []*object.Commit - // get current head commit sha, default is master branch - // check branch, if not master, checkout to branch's head - commitOid, err := r.repo.Head() - if err != nil { - return nil, err - } - // get head commit object and add into commitList - commit, err := r.repo.CommitObject(commitOid.Hash()) - if err != nil { - return nil, err - } - commitList = append(commitList, commit) - // if current head has parents, get parent commit sha - for commit != nil && commit.NumParents() > 0 { - parentCommit, err := commit.Parent(0) - if err != nil { - return nil, err - } - commit, err = r.repo.CommitObject(parentCommit.Hash) - if err != nil { - return nil, err - } - commitList = append(commitList, commit) - } - // reverse commitList - // use slices.Reverse(commitList) in higher golang version. - for i, j := 0, len(commitList)-1; i < j; i, j = i+1, j-1 { - commitList[i], commitList[j] = commitList[j], commitList[i] - } - return commitList, nil -} - -func (r *GogitRepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { - commitList, err := r.GetCommitList(subtaskCtx) - if err != nil { - return err - } - if err := r.storeRepoSnapshot(subtaskCtx, commitList); err != nil { - return err - } - // fixme: collecting CommitLineChange is not implemented. - // There is no way to get such information with go-git, and table commit_line_change is not used by any dashboards - // So we just ignore it. - return nil -} diff --git a/backend/plugins/gitextractor/parser/repo_gogit_test.go b/backend/plugins/gitextractor/parser/repo_gogit_test.go deleted file mode 100644 index a4dfa321b3e..00000000000 --- a/backend/plugins/gitextractor/parser/repo_gogit_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// buildTestRepo creates a repository on disk with a known shape: 3 commits on -// the default branch, one lightweight tag and a second branch which is checked -// out at the end, so HEAD does not point at the default branch. -func buildTestRepo(t *testing.T) string { - t.Helper() - dir := t.TempDir() - - repo, err := gogit.PlainInit(dir, false) - require.NoError(t, err) - wt, err := repo.Worktree() - require.NoError(t, err) - - sig := func(n int) *object.Signature { - return &object.Signature{ - Name: "DevLake Test", - Email: "test@devlake.apache.org", - When: time.Date(2026, 1, 1, 0, 0, n, 0, time.UTC), - } - } - - var last plumbing.Hash - for i := 1; i <= 3; i++ { - name := filepath.Join(dir, "file.txt") - require.NoError(t, os.WriteFile(name, []byte{byte('a' + i)}, 0600)) - _, err = wt.Add("file.txt") - require.NoError(t, err) - last, err = wt.Commit("commit", &gogit.CommitOptions{Author: sig(i)}) - require.NoError(t, err) - } - - _, err = repo.CreateTag("v1.0.0", last, nil) - require.NoError(t, err) - - require.NoError(t, wt.Checkout(&gogit.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName("feature"), - Create: true, - })) - - return dir -} - -func TestGogitRepoCollectorCounts(t *testing.T) { - dir := buildTestRepo(t) - - // store and logger are only used by the Collect* subtasks, not by the - // counting helpers exercised here. - collector, err := NewGogitRepoCollector(dir, "test-repo", nil, nil) - require.Nil(t, err) - - ctx := context.Background() - - commits, cErr := collector.CountCommits(ctx) - assert.NoError(t, cErr) - assert.Equal(t, 3, commits) - - tags, tErr := collector.CountTags(ctx) - assert.NoError(t, tErr) - assert.Equal(t, 1, tags) - - // HEAD is on "feature", so only the default branch is counted. - branches, bErr := collector.CountBranches(ctx) - assert.NoError(t, bErr) - assert.Equal(t, 1, branches) -} - -func TestGogitRepoCollectorCountsHonourCancelledContext(t *testing.T) { - dir := buildTestRepo(t) - - collector, err := NewGogitRepoCollector(dir, "test-repo", nil, nil) - require.Nil(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, cErr := collector.CountCommits(ctx) - assert.ErrorIs(t, cErr, context.Canceled) - - _, tErr := collector.CountTags(ctx) - assert.ErrorIs(t, tErr, context.Canceled) - - _, bErr := collector.CountBranches(ctx) - assert.ErrorIs(t, bErr, context.Canceled) -} - -func TestNewGogitRepoCollectorRejectsNonRepo(t *testing.T) { - _, err := NewGogitRepoCollector(t.TempDir(), "test-repo", nil, nil) - assert.NotNil(t, err) -} - diff --git a/backend/plugins/gitextractor/parser/repo_libgit2.go b/backend/plugins/gitextractor/parser/repo_libgit2.go deleted file mode 100644 index ad9b515b4bf..00000000000 --- a/backend/plugins/gitextractor/parser/repo_libgit2.go +++ /dev/null @@ -1,698 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/apache/devlake/core/dal" - "github.com/apache/devlake/core/errors" - "github.com/apache/devlake/core/log" - "github.com/apache/devlake/core/models/domainlayer" - "github.com/apache/devlake/core/models/domainlayer/code" - "github.com/apache/devlake/core/plugin" - "github.com/apache/devlake/plugins/gitextractor/models" - - git "github.com/libgit2/git2go/v34" -) - -var _ RepoCollector = (*Libgit2RepoCollector)(nil) - -var TypeNotMatchError = "the requested type does not match the type in the ODB" - -type Libgit2RepoCollector struct { - id string - logger log.Logger - - store models.Store - repo *git.Repository - cleanup func() -} - -func NewLibgit2RepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*Libgit2RepoCollector, errors.Error) { - repo, err := git.OpenRepository(localDir) - if err != nil { - return nil, errors.Convert(err) - } - return &Libgit2RepoCollector{ - id: repoId, - logger: logger, - store: store, - repo: repo, - }, nil -} - -func (r *Libgit2RepoCollector) SetCleanUp(f func()) error { - if f != nil { - r.cleanup = f - } - return nil -} - -// CollectAll The main parser subtask -func (r *Libgit2RepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { - subtaskCtx.SetProgress(0, -1) - err := r.CollectTags(subtaskCtx) - if err != nil { - return err - } - err = r.CollectBranches(subtaskCtx) - if err != nil { - return err - } - err = r.CollectCommits(subtaskCtx) - if err != nil { - return err - } - opt := subtaskCtx.GetData().(*GitExtractorTaskData).Options - if !*opt.SkipCommitStat { - return r.CollectDiffLine(subtaskCtx) - } - return nil -} - -// Close resources -func (r *Libgit2RepoCollector) Close(ctx context.Context) error { - defer func() { - if r.cleanup != nil { - r.cleanup() - } - }() - return r.store.Close() -} - -// CountTags Count git tags subtask -func (r *Libgit2RepoCollector) CountTags(ctx context.Context) (int, error) { - tags, err := r.repo.Tags.List() - if err != nil { - return 0, errors.Convert(err) - } - return len(tags), nil -} - -// CountBranches count the number of branches in a git repo -func (r *Libgit2RepoCollector) CountBranches(ctx context.Context) (int, error) { - var branchIter *git.BranchIterator - branchIter, err := r.repo.NewBranchIterator(git.BranchAll) - if err != nil { - return 0, errors.Convert(err) - } - count := 0 - err = branchIter.ForEach(func(branch *git.Branch, branchType git.BranchType) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if branch.IsBranch() || branch.IsRemote() { - isHead, _ := branch.IsHead() - if !isHead { - count++ - } - } - return nil - }) - return count, errors.Convert(err) -} - -// CountCommits count the number of commits in a git repo -func (r *Libgit2RepoCollector) CountCommits(ctx context.Context) (int, error) { - odb, err := r.repo.Odb() - if err != nil { - return 0, errors.Convert(err) - } - count := 0 - err = odb.ForEach(func(id *git.Oid) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - commit, e := r.repo.LookupCommit(id) - if e != nil && e.Error() != TypeNotMatchError { - return errors.Convert(e) - } - if commit != nil { - count++ - } - return nil - }) - return count, errors.Convert(err) -} - -// CollectTags Collect Tags data -func (r *Libgit2RepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { - return errors.Convert(r.repo.Tags.Foreach(func(name string, id *git.Oid) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - var err1 error - var tag *git.Tag - var tagCommit string - tag, err1 = r.repo.LookupTag(id) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - if tag != nil { - tagCommit = tag.TargetId().String() - } else { - tagCommit = id.String() - } - r.logger.Debug("tagCommit: %s", tagCommit) - if tagCommit != "" { - ref := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: tagCommit, - RefType: TAG, - } - err1 = r.store.Refs(ref) - if err1 != nil { - return err1 - } - subtaskCtx.IncProgress(1) - } - return nil - })) -} - -// CollectBranches Collect branch data -func (r *Libgit2RepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { - var repoInter *git.BranchIterator - repoInter, err := r.repo.NewBranchIterator(git.BranchAll) - if err != nil { - return errors.Convert(err) - } - return errors.Convert(repoInter.ForEach(func(branch *git.Branch, branchType git.BranchType) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - if branch.IsBranch() || branch.IsRemote() { - name, err1 := branch.Name() - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - var sha string - if oid := branch.Target(); oid != nil { - sha = oid.String() - } - ref := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: sha, - RefType: BRANCH, - } - // commit sha may be an empty string, it's unexpected. - ref.IsDefault, err1 = branch.IsHead() - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - err1 = r.store.Refs(ref) - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - subtaskCtx.IncProgress(1) - return nil - } - return nil - })) -} - -// CollectCommits Collect data from each commit, we can also get the diff line -func (r *Libgit2RepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) error { - taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options - opts, err := getDiffOpts() - if err != nil { - return err - } - db := subtaskCtx.GetDal() - components := make([]code.Component, 0) - err = db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) - if err != nil { - return err - } - componentMap := make(map[string]*regexp.Regexp) - for _, component := range components { - componentMap[component.Name] = regexp.MustCompile(component.PathRegex) - } - odb, err := errors.Convert01(r.repo.Odb()) - if err != nil { - return err - } - return errors.Convert(odb.ForEach(func(id *git.Oid) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - commit, err1 := r.repo.LookupCommit(id) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - if commit == nil { - return nil - } - var parent *git.Commit - if commit.ParentCount() > 0 { - parent = commit.Parent(0) - // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. - // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting - // the correct addition/deletion data in the database with an absurdly large addition number. - if parent == nil { - r.logger.Info("skip commit %s because it has no parent commit", commit.Id().String()) - return nil - } - } - commitSha := commit.Id().String() - r.logger.Debug("process commit: %s", commitSha) - c := &code.Commit{ - Sha: commitSha, - Message: commit.Message(), - } - author := commit.Author() - if author != nil { - c.AuthorName = author.Name - c.AuthorEmail = author.Email - c.AuthorId = author.Email - c.AuthoredDate = author.When - } - committer := commit.Committer() - if committer != nil { - c.CommitterName = committer.Name - c.CommitterEmail = committer.Email - c.CommitterId = committer.Email - c.CommittedDate = committer.When - } - err = r.storeParentCommits(commitSha, commit) - if err != nil { - return err - } - - if !*taskOpts.SkipCommitStat { - var stats *git.DiffStats - var addIncluded, delIncluded int - if stats, addIncluded, delIncluded, err = r.getDiffComparedToParent(taskOpts, c.Sha, commit, parent, opts, componentMap); err != nil { - return err - } - r.logger.Debug("state: %#+v\n", stats.Deletions()) - c.Additions += addIncluded - c.Deletions += delIncluded - } - - err = r.store.Commits(c) - if err != nil { - return err - } - repoCommit := &code.RepoCommit{ - RepoId: r.id, - CommitSha: c.Sha, - } - err = r.store.RepoCommits(repoCommit) - if err != nil { - return err - } - subtaskCtx.IncProgress(1) - return nil - })) -} - -func (r *Libgit2RepoCollector) storeParentCommits(commitSha string, commit *git.Commit) errors.Error { - var commitParents []*code.CommitParent - for i := uint(0); i < commit.ParentCount(); i++ { - parent := commit.Parent(i) - if parent != nil { - if parentId := parent.Id(); parentId != nil { - commitParents = append(commitParents, &code.CommitParent{ - CommitSha: commitSha, - ParentCommitSha: parentId.String(), - }) - } - } - } - return r.store.CommitParents(commitParents) -} - -func (r *Libgit2RepoCollector) getDiffComparedToParent(taskOpts *GitExtractorOptions, commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, int, int, errors.Error) { - var err error - var parentTree, tree *git.Tree - if parent != nil { - parentTree, err = parent.Tree() - } - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - tree, err = commit.Tree() - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - var diff *git.Diff - diff, err = r.repo.DiffTreeToTree(parentTree, tree, opts) - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - // build excluded extension set - excluded := map[string]struct{}{} - for _, ext := range taskOpts.ExcludeFileExtensions { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - if !*taskOpts.SkipCommitFiles { - err = r.storeCommitFilesFromDiff(commitSha, diff, componentMap, excluded) - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - } - var stats *git.DiffStats - stats, err = diff.Stats() - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - // calculate included totals with exclusions - addIncluded := 0 - delIncluded := 0 - if len(excluded) == 0 { - addIncluded = stats.Insertions() - delIncluded = stats.Deletions() - return stats, addIncluded, delIncluded, nil - } - _ = diff.ForEach(func(file git.DiffDelta, progress float64) (git.DiffForEachHunkCallback, error) { - // choose path to check based on delta status; for deletions use old path - pathForCheck := file.NewFile.Path - if file.Status == git.DeltaDeleted || pathForCheck == "" { - pathForCheck = file.OldFile.Path - } - lower := strings.ToLower(pathForCheck) - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - // skip all lines for excluded files - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { return nil }, nil - }, nil - } - } - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { - if line.Origin == git.DiffLineAddition { - addIncluded += line.NumLines - } - if line.Origin == git.DiffLineDeletion { - delIncluded += line.NumLines - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - return stats, addIncluded, delIncluded, nil -} - -func (r *Libgit2RepoCollector) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp, excluded map[string]struct{}) errors.Error { - var commitFile *code.CommitFile - var commitFileComponent *code.CommitFileComponent - var err error - err = diff.ForEach(func(file git.DiffDelta, progress float64) ( - git.DiffForEachHunkCallback, error) { - if commitFile != nil { - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - return nil, err - } - } - - // skip files by extension if configured - if len(excluded) > 0 { - pathForCheck := file.NewFile.Path - if file.Status == git.DeltaDeleted || pathForCheck == "" { - pathForCheck = file.OldFile.Path - } - lower := strings.ToLower(pathForCheck) - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - // skip this file entirely - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { return nil }, nil - }, nil - } - } - } - - commitFile = new(code.CommitFile) - commitFile.CommitSha = commitSha - // prefer new path; for deletions fall back to old path - commitFile.FilePath = file.NewFile.Path - if commitFile.FilePath == "" { - commitFile.FilePath = file.OldFile.Path - } - - // With some long path,the varchar(255) was not enough both ID and file_path - // So we use the hash to compress the path in ID and add length of file_path. - // Use commitSha and the sha256 of FilePath to create id - shaFilePath := sha256.New() - shaFilePath.Write([]byte(commitFile.FilePath)) - commitFile.Id = commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) - - commitFileComponent = new(code.CommitFileComponent) - for component, reg := range componentMap { - if reg.MatchString(commitFile.FilePath) { - commitFileComponent.ComponentName = component - break - } - } - commitFileComponent.CommitFileId = commitFile.Id - if commitFileComponent.ComponentName == "" { - commitFileComponent.ComponentName = "Default" - } - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { - if line.Origin == git.DiffLineAddition { - commitFile.Additions += line.NumLines - } - if line.Origin == git.DiffLineDeletion { - commitFile.Deletions += line.NumLines - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - if commitFileComponent != nil { - err = r.store.CommitFileComponents(commitFileComponent) - if err != nil { - r.logger.Error(err, "CommitFileComponents error") - } - } - if commitFile != nil { - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - } - } - return errors.Convert(err) -} - -// CollectDiffLine get line diff data from a specific branch -func (r *Libgit2RepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { - //Using this subtask,we can get every line change in every commit. - //We maintain a snapshot structure to get which commit each deleted line belongs to - snapshot := make(map[string] /*file path*/ *models.FileBlame) - repo := r.repo - //step 1. get the reverse commit list - commitList := make([]git.Commit, 0) - // get current head commit sha, default is master branch - // check branch, if not master, checkout to branch's head - commitOid, err1 := repo.Head() - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - //get head commit object and add into commitList - commit, err1 := repo.LookupCommit(commitOid.Target()) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - commitList = append(commitList, *commit) - // if current head has parents, get parent commitsha - for commit != nil && commit.ParentCount() > 0 { - pid := commit.ParentId(0) - commit, err1 = repo.LookupCommit(pid) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - commitList = append(commitList, *commit) - } - // reverse commitList - for i, j := 0, len(commitList)-1; i < j; i, j = i+1, j-1 { - commitList[i], commitList[j] = commitList[j], commitList[i] - } - //step 2. get the diff of each commit - // for each commit, get the diff - for _, commitsha := range commitList { - curcommit, err := repo.LookupCommit(commitsha.Id()) - if err != nil { - return errors.Convert(err) - } - if curcommit.ParentCount() == 0 || curcommit.ParentCount() > 0 { - var parentTree, tree *git.Tree - tree, err = curcommit.Tree() - if err != nil { - return errors.Convert(err) - } - var diff *git.Diff - //FIXME error type convert - opts, err := git.DefaultDiffOptions() - opts.NotifyCallback = func(diffSoFar *git.Diff, delta git.DiffDelta, matchedPathSpec string) error { - return nil - } - if err != nil { - return errors.Convert(err) - } - if curcommit.ParentCount() > 0 { - parent := curcommit.Parent(0) - parentTree, err = parent.Tree() - } - diff, err = repo.DiffTreeToTree(parentTree, tree, &opts) - if err != nil { - return errors.Convert(err) - } - deleted := make(models.DiffLines, 0) - added := make(models.DiffLines, 0) - var lastFile string - lastFile = "" - err = diff.ForEach(func(file git.DiffDelta, progress float64) (git.DiffForEachHunkCallback, error) { - // if it doesn't exist in snapshot, create a new one - if _, ok := snapshot[file.OldFile.Path]; !ok { - fileBlame, err := models.NewFileBlame() - if err != nil { - r.logger.Info("Create FileBlame Error") - return nil, err - } - snapshot[file.OldFile.Path] = (*models.FileBlame)(fileBlame) - } - if lastFile == "" { - lastFile = file.NewFile.Path - } else if lastFile != file.NewFile.Path { - updateSnapshotFileBlame(curcommit, deleted, added, lastFile, snapshot) - // reset the deleted and added,last_file now is current file - deleted = make([]git.DiffLine, 0) - added = make([]git.DiffLine, 0) - lastFile = file.NewFile.Path - } - hunkNum := 0 - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - hunkNum++ - return func(line git.DiffLine) error { - commitLineChange := &code.CommitLineChange{} - commitLineChange.CommitSha = curcommit.Id().String() - commitLineChange.ChangedType = line.Origin.String() - commitLineChange.LineNoNew = line.NewLineno - commitLineChange.LineNoOld = line.OldLineno - commitLineChange.OldFilePath = file.OldFile.Path - commitLineChange.NewFilePath = file.NewFile.Path - commitLineChange.HunkNum = hunkNum - commitLineChange.Id = curcommit.Id().String() + ":" + file.NewFile.Path + ":" + strconv.Itoa(line.OldLineno) + ":" + strconv.Itoa(line.NewLineno) - if line.Origin == git.DiffLineAddition { - added = append(added, line) - } else if line.Origin == git.DiffLineDeletion { - fb := snapshot[file.OldFile.Path] - l := fb.Find(line.OldLineno) - if l != nil && l.Value != nil { - temp := snapshot[file.OldFile.Path].Find(line.OldLineno) - commitLineChange.PrevCommit = temp.Value.(string) - } else { - r.logger.Info("err", file.OldFile.Path, line.OldLineno, curcommit.Id().String()) - } - deleted = append(deleted, line) - } - err = r.store.CommitLineChange(commitLineChange) - if err != nil { - return errors.Convert(err) - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - if err != nil { - return errors.Convert(err) - } - //finally,process the last file in diff - updateSnapshotFileBlame(curcommit, deleted, added, lastFile, snapshot) - } - } - r.logger.Info("line change collect success") - db := subtaskCtx.GetDal() - err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id)) - if err != nil { - return errors.Convert(err) - } - for fp := range snapshot { - temp := snapshot[fp] - count := 0 - for e := temp.Lines.Front(); e != nil; e = e.Next() { - count++ - snapshotLine := &code.RepoSnapshot{} - snapshotLine.RepoId = r.id - snapshotLine.LineNo = count - snapshotLine.CommitSha = e.Value.(string) - snapshotLine.FilePath = fp - err := r.store.RepoSnapshot(snapshotLine) - if err != nil { - r.logger.Info("error") - return err - } - } - - } - - r.logger.Info("collect snapshot finished") - return nil -} - -func updateSnapshotFileBlame(currentCommit *git.Commit, deleted models.DiffLines, added models.DiffLines, lastFile string, snapshot map[string]*models.FileBlame) { - sort.Sort(deleted) - for _, line := range deleted { - snapshot[lastFile].RemoveLine(line.OldLineno) - } - for _, line := range added { - snapshot[lastFile].AddLine(line.NewLineno, currentCommit.Id().String()) - } -} - -func getDiffOpts() (*git.DiffOptions, errors.Error) { - opts, err := git.DefaultDiffOptions() - if err != nil { - return nil, errors.Convert(err) - } - opts.NotifyCallback = func(diffSoFar *git.Diff, delta git.DiffDelta, matchedPathSpec string) error { - return nil - } - return &opts, nil -} diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index bdfdbd2ae1b..707c27b640f 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -41,7 +41,6 @@ type GitExtractorOptions struct { PrivateKey string `json:"privateKey" mapstructure:"privateKey"` Passphrase string `json:"passphrase" mapstructure:"passphrase"` Proxy string `json:"proxy" mapstructure:"proxy"` - UseGoGit *bool `json:"useGoGit" mapstructure:"useGoGit"` SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` diff --git a/backend/plugins/gitextractor/parser/testrepo_test.go b/backend/plugins/gitextractor/parser/testrepo_test.go new file mode 100644 index 00000000000..400acd63730 --- /dev/null +++ b/backend/plugins/gitextractor/parser/testrepo_test.go @@ -0,0 +1,316 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// This file builds deterministic git repositories with the git command line. +// The collector tests use them as fixtures so that the fixtures do not depend +// on any of the git libraries under test. + +const ( + testAuthorName = "DevLake Test" + testAuthorEmail = "test@devlake.apache.org" +) + +var ( + testTimeBase = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + testTimeC1 = testTimeBase + testTimeC2 = testTimeBase.Add(1 * time.Hour) + testTimeC3 = testTimeBase.Add(2 * time.Hour) + testTimeC4 = testTimeBase.Add(3 * time.Hour) + testTimeC5 = testTimeBase.Add(4 * time.Hour) +) + +func requireGit(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } +} + +// gitFixture is a scriptable git repository. Commands run with an isolated +// git configuration and a fixed identity so the fixtures are reproducible. +type gitFixture struct { + t *testing.T + dir string +} + +func newGitFixture(t *testing.T) *gitFixture { + t.Helper() + requireGit(t) + f := newGitFixtureAt(t, t.TempDir()) + f.git("init", "-q", "-b", "main") + return f +} + +func newGitFixtureAt(t *testing.T, dir string) *gitFixture { + t.Helper() + requireGit(t) + return &gitFixture{t: t, dir: dir} +} + +func (f *gitFixture) env(when time.Time) []string { + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+testAuthorName, + "GIT_AUTHOR_EMAIL="+testAuthorEmail, + "GIT_COMMITTER_NAME="+testAuthorName, + "GIT_COMMITTER_EMAIL="+testAuthorEmail, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL="+os.DevNull, + "GIT_TERMINAL_PROMPT=0", + "TZ=UTC", + // keep the fixtures independent of the host user configuration + "HOME="+f.dir, + ) + if !when.IsZero() { + date := when.UTC().Format(time.RFC3339) + env = append(env, "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date) + } + return env +} + +func (f *gitFixture) git(args ...string) string { + f.t.Helper() + return f.gitAt(time.Time{}, args...) +} + +func (f *gitFixture) gitAt(when time.Time, args ...string) string { + f.t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = f.dir + cmd.Env = f.env(when) + output, err := cmd.CombinedOutput() + require.NoError(f.t, err, "git %v in %s: %s", args, f.dir, output) + return strings.TrimSpace(string(output)) +} + +func (f *gitFixture) head() string { + f.t.Helper() + return f.git("rev-parse", "HEAD") +} + +func (f *gitFixture) writeFile(rel, content string) { + f.t.Helper() + full := filepath.Join(f.dir, rel) + require.NoError(f.t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(f.t, os.WriteFile(full, []byte(content), 0o644)) +} + +func (f *gitFixture) chmod(rel string, mode os.FileMode) { + f.t.Helper() + require.NoError(f.t, os.Chmod(filepath.Join(f.dir, rel), mode)) +} + +func (f *gitFixture) removeFile(rel string) { + f.t.Helper() + require.NoError(f.t, os.Remove(filepath.Join(f.dir, rel))) +} + +// commit stages everything and commits with the fixture identity at when. +func (f *gitFixture) commit(when time.Time, message string) string { + f.t.Helper() + f.git("add", "-A") + f.gitAt(when, "commit", "-q", "-m", message) + return f.head() +} + +func (f *gitFixture) checkout(branch string, create bool) { + f.t.Helper() + if create { + f.git("checkout", "-q", "-b", branch) + return + } + f.git("checkout", "-q", branch) +} + +func (f *gitFixture) merge(when time.Time, message, branch string) string { + f.t.Helper() + f.gitAt(when, "merge", "-q", "--no-ff", "-m", message, branch) + return f.head() +} + +func (f *gitFixture) tag(name, target string, annotated bool) { + f.t.Helper() + if annotated { + f.git("tag", "-a", name, target, "-m", "release "+name) + return + } + f.git("tag", name, target) +} + +// bareClone clones the fixture into a new bare repository, which is the shape +// the gitextractor produces before collecting. +func (f *gitFixture) bareClone() string { + f.t.Helper() + dir := filepath.Join(f.t.TempDir(), "repo.git") + f.git("clone", "-q", "--bare", f.dir, dir) + return dir +} + +// shallowBareClone creates a depth-1 bare clone, so the tip commit is a +// shallow boundary whose parent is missing from the object database. +func (f *gitFixture) shallowBareClone() string { + f.t.Helper() + dir := filepath.Join(f.t.TempDir(), "shallow.git") + f.git("clone", "-q", "--bare", "--depth=1", "file://"+f.dir, dir) + return dir +} + +// canonicalRepo is the repository used by the collector contract tests. +// +// main: c1 ---- c3 ---- c4 ---- c5 (merge) +// \ / +// feature: c2 ---------------- +// +// c1 adds a.txt (3 lines), a file in a directory with a space, a binary file +// and an empty file. c2 adds a file on the feature branch. c3 rewrites a line +// in a.txt, appends a line and deletes the empty file. c4 only changes the file +// mode of a.txt and rewrites the binary file. c5 merges feature. +// c1 carries the lightweight tag v1.0.0, c3 the annotated tag v2.0.0. +type canonicalRepo struct { + Dir string + C1 string + C2 string + C3 string + C4 string + C5 string +} + +func buildCanonicalRepo(t *testing.T) canonicalRepo { + t.Helper() + f := newGitFixture(t) + + f.writeFile("a.txt", "l1\nl2\nl3\n") + f.writeFile("dir with space/b.txt", "b1\n") + f.writeFile("bin.dat", "before\x00binary\n") + f.writeFile("empty.txt", "") + c1 := f.commit(testTimeC1, "c1: initial import") + + f.checkout("feature", true) + f.writeFile("feature.txt", "f1\n") + c2 := f.commit(testTimeC2, "c2: add feature file") + + f.checkout("main", false) + f.writeFile("a.txt", "l1\nL2\nl3\nl4\n") + f.removeFile("empty.txt") + c3 := f.commit(testTimeC3, "c3: reword and extend a.txt") + + f.chmod("a.txt", 0o755) + f.writeFile("bin.dat", "after\x00binary\n") + c4 := f.commit(testTimeC4, "c4: mode and binary change") + + c5 := f.merge(testTimeC5, "c5: merge feature", "feature") + + f.tag("v1.0.0", c1, false) + f.tag("v2.0.0", c3, true) + + return canonicalRepo{ + Dir: f.bareClone(), + C1: c1, + C2: c2, + C3: c3, + C4: c4, + C5: c5, + } +} + +// buildLinearRepo creates a repository without merges for the diff line tests. +// +// c1 adds a.txt and b.txt, c2 rewrites a line in a.txt, c3 appends a line to +// a.txt and rewrites a line in b.txt, c4 deletes b.txt and adds c.txt. +type linearRepo struct { + Dir string + C1 string + C2 string + C3 string + C4 string +} + +func buildLinearRepo(t *testing.T) linearRepo { + t.Helper() + f := newGitFixture(t) + + f.writeFile("a.txt", "l1\nl2\nl3\n") + f.writeFile("b.txt", "b1\nb2\n") + c1 := f.commit(testTimeC1, "c1") + + f.writeFile("a.txt", "l1\nL2\nl3\n") + c2 := f.commit(testTimeC2, "c2") + + f.writeFile("a.txt", "l1\nL2\nl3\nl4\n") + f.writeFile("b.txt", "b1\nB2\n") + c3 := f.commit(testTimeC3, "c3") + + f.removeFile("b.txt") + f.writeFile("c.txt", "c1\nc2\n") + c4 := f.commit(testTimeC4, "c4") + + return linearRepo{ + Dir: f.bareClone(), + C1: c1, + C2: c2, + C3: c3, + C4: c4, + } +} + +// blameAtHEAD returns file -> line number -> commit sha for every file in HEAD, +// using the git command line as an independent oracle. +func blameAtHEAD(t *testing.T, dir string) map[string]map[int]string { + t.Helper() + repo := newGitFixtureAt(t, dir) + blames := make(map[string]map[int]string) + files := strings.Split(strings.TrimRight(repo.git("ls-tree", "-r", "--name-only", "-z", "HEAD"), "\x00"), "\x00") + for _, file := range files { + if file == "" { + continue + } + out := repo.git("blame", "-l", "--line-porcelain", "HEAD", "--", file) + lines := make(map[int]string) + for _, line := range strings.Split(out, "\n") { + // The header of the first line of each group carries the group + // size, later headers do not. Content lines are tab-prefixed. + if strings.HasPrefix(line, "\t") { + continue + } + fields := strings.Fields(line) + if len(fields) < 3 || len(fields[0]) != 40 { + continue + } + finalLine, err := strconv.Atoi(fields[2]) + if err != nil { + continue + } + lines[finalLine] = fields[0] + } + blames[file] = lines + } + return blames +} diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go index 76013e28e96..0c218e9d80d 100644 --- a/backend/plugins/gitextractor/tasks/repo_cloner.go +++ b/backend/plugins/gitextractor/tasks/repo_cloner.go @@ -69,13 +69,7 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { if repoCloner.IsIncremental() { storage.SetIncrementalMode(repoCloner.IsIncremental()) } - // We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance. - var repoCollector parser.RepoCollector - if *taskData.Options.UseGoGit { - repoCollector, err = parser.NewGogitRepoCollector(localDir, op.RepoId, storage, logger) - } else { - repoCollector, err = parser.NewLibgit2RepoCollector(localDir, op.RepoId, storage, logger) - } + repoCollector, err := parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) if err != nil { return err } diff --git a/backend/plugins/kiro/impl/table_info_test.go b/backend/plugins/kiro/impl/table_info_test.go index 90598fd1e12..3291b12c3ef 100644 --- a/backend/plugins/kiro/impl/table_info_test.go +++ b/backend/plugins/kiro/impl/table_info_test.go @@ -24,10 +24,9 @@ import ( ) // The repo-wide plugins/table_info_test.go performs this same check for every -// plugin, but building that package requires gitextractor and therefore a -// specific libgit2. Running it here as well keeps the feedback local: a model -// added without registering it in GetTablesInfo fails immediately rather than -// only in CI. +// plugin. Running it here as well keeps the feedback local: a model added +// without registering it in GetTablesInfo fails immediately rather than only +// in CI. func TestKiroTableInfo(t *testing.T) { checker := unithelper.NewTableInfoChecker(unithelper.TableInfoCheckerConfig{}) checker.FeedIn("../models", Kiro{}.GetTablesInfo) diff --git a/backend/python/DevelopmentSetup.md b/backend/python/DevelopmentSetup.md index c831313c8de..c171bd81dbf 100644 --- a/backend/python/DevelopmentSetup.md +++ b/backend/python/DevelopmentSetup.md @@ -16,7 +16,7 @@ limitations under the License. --> # Python prerequisites for `make e2e-test` -`make e2e-test` also requires a test database and, for some service tests, `libgit2`. See +`make e2e-test` also requires a test database and `git` for the service tests. See [`backend/test/Readme.md`](../test/Readme.md) for the full local E2E test flow. For the Python plugin tests to run properly, the following steps must be taken: diff --git a/backend/scripts/install-libgit2.sh b/backend/scripts/install-libgit2.sh deleted file mode 100755 index 7f6ced6184c..00000000000 --- a/backend/scripts/install-libgit2.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -LIBGIT2_VERSION=1.5.0 -LIBGIT2_SHA256=8de872a0f201b33d9522b817c92e14edb4efad18dae95cf156cf240b2efff93e -LIBGIT2_URL=https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz - -current_version=$(pkg-config --modversion libgit2 2>/dev/null || true) -case "$current_version" in - 1.5.*) - echo "libgit2 $current_version is already installed" - exit 0 - ;; -esac - -if [ "$(id -u)" -ne 0 ]; then - echo "install-libgit2.sh must run as root" >&2 - exit 1 -fi - -build_dir=$(mktemp -d) -trap 'rm -rf "$build_dir"' EXIT HUP INT TERM - -export DEBIAN_FRONTEND=noninteractive - -# Debian releases are pulled from security.debian.org/deb.debian.org only -# while they have active (LTS) support. Once a release goes EOL, its -# packages move to archive.debian.org and the live mirrors 404. Repoint -# apt at the archive for releases we know are EOL so this keeps working -# without depending on the CI image being rebuilt on a newer base. -eol_codenames="bullseye" -codename=$(. /etc/os-release && echo "$VERSION_CODENAME") -case " $eol_codenames " in - *" $codename "*) - echo "Debian $codename is EOL; repointing apt sources at archive.debian.org" - for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list; do - [ -f "$f" ] || continue - sed -i \ - -e "s|http://deb.debian.org/debian|http://archive.debian.org/debian|g" \ - -e "s|http://security.debian.org/debian-security|http://archive.debian.org/debian-security|g" \ - "$f" - done - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until - ;; -esac - -apt_update_and_install() { - apt-get update - apt-get install -y --no-install-recommends \ - ca-certificates \ - cmake \ - curl \ - gcc \ - libssh2-1-dev \ - libssl-dev \ - make \ - pkg-config \ - zlib1g-dev -} - -n=0 -until apt_update_and_install; do - n=$((n + 1)) - if [ "$n" -ge 3 ]; then - echo "apt-get update/install failed after $n attempts" >&2 - exit 1 - fi - echo "apt-get update/install failed, retrying ($n/3)..." >&2 - apt-get clean - sleep 5 -done - -archive="$build_dir/libgit2.tar.gz" -curl --fail --location --retry 3 "$LIBGIT2_URL" --output "$archive" -echo "$LIBGIT2_SHA256 $archive" | sha256sum --check --strict - -mkdir "$build_dir/source" -tar -xzf "$archive" -C "$build_dir/source" --strip-components=1 - -multiarch=$(gcc -print-multiarch) -libdir=lib -if [ -n "$multiarch" ]; then - libdir="lib/$multiarch" -fi - -cmake \ - -S "$build_dir/source" \ - -B "$build_dir/build" \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR="$libdir" \ - -DCMAKE_INSTALL_PREFIX=/usr -cmake --build "$build_dir/build" --parallel "$(getconf _NPROCESSORS_ONLN)" -cmake --install "$build_dir/build" -ldconfig - -export PKG_CONFIG_PATH="/usr/$libdir/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" -installed_version=$(pkg-config --modversion libgit2) -if [ "$installed_version" != "$LIBGIT2_VERSION" ]; then - echo "expected libgit2 $LIBGIT2_VERSION, found $installed_version" >&2 - exit 1 -fi - -if [ -n "${GITHUB_ENV:-}" ]; then - echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> "$GITHUB_ENV" -fi - -rm -rf /var/lib/apt/lists/* -echo "installed libgit2 $installed_version" diff --git a/backend/test/Readme.md b/backend/test/Readme.md index 87bfcfa30b2..70931816ee3 100644 --- a/backend/test/Readme.md +++ b/backend/test/Readme.md @@ -32,15 +32,8 @@ go run ./test/init.go make e2e-test ``` -The service E2E tests compile the `gitextractor` plugin, which links against `libgit2`. If `libgit2` is installed in -a non-standard location, expose it through `pkg-config` and add an rpath so the test binary can load the dynamic -library at runtime: - -```bash -export PKG_CONFIG_PATH=/path/to/libgit2/build -export CGO_LDFLAGS='-L/path/to/libgit2/build -Wl,-rpath,/path/to/libgit2/build' -make e2e-test -``` +The service E2E tests run the `gitextractor` plugin, which reads repositories +with the `git` command line. Make sure `git` (2.31 or newer) is installed. ### Planning-failure database regression test diff --git a/devops/docker/lake-builder/Dockerfile b/devops/docker/lake-builder/Dockerfile index 95cd0544eef..64c95bbebe3 100644 --- a/devops/docker/lake-builder/Dockerfile +++ b/devops/docker/lake-builder/Dockerfile @@ -14,51 +14,14 @@ # limitations under the License. # -# Debian 11 (bullseye) reached the end of its LTS period on 2026-08-31 and its -# security suite is no longer served, so `apt upgrade` fails there. All stages -# use Debian 12 (bookworm), matching the runtime image in backend/Dockerfile. -FROM --platform=linux/amd64 debian:bookworm AS debian-amd64 -RUN apt-get -y update && apt -y upgrade &&\ - apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - -# This stage only cross-builds libgit2, it does not need a Go toolchain. -FROM --platform=linux/amd64 debian:bookworm AS builder - -# Base dependencies -RUN apt-get -y update && apt -y upgrade &&\ - apt-get install -y ca-certificates wget gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev \ - gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu - -COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include -COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu -COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu - -## Build libs/headers needed for gitextractor plugin -RUN \ - mkdir -p /tmp/build/x86_64 && cd /tmp/build/x86_64 && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTS=OFF -DCMAKE_SYSROOT=/rootfs-amd64 -DCMAKE_INSTALL_PREFIX=/usr/local/deps/x86_64 &&\ - make -j install &&\ - mkdir -p /tmp/deps &&\ - cp *libgit2* /tmp/deps/ &&\ - cp -r ../include /tmp/deps/include - # Same Python line as the runtime image in backend/Dockerfile. FROM python:3.11-slim-bookworm RUN apt -y update && apt -y upgrade && apt -y install tzdata make tar curl gcc g++ pkg-config git \ - libssh2-1 zlib1g libffi-dev \ + zlib1g libffi-dev \ default-libmysqlclient-dev \ libpq-dev -# Install Libs/Headers from previous stage -COPY --from=builder /tmp/deps/*.so* /usr/lib/ -COPY --from=builder /tmp/deps/*.pc /usr/lib/x86_64-linux-gnu/pkgconfig/ -COPY --from=builder /tmp/deps/include/ /usr/include/ -ENV PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig - # Install Golang # The previous installer was fetched from `git.io`, which GitHub shut down in # 2022, so the image could no longer be rebuilt. Fetch the official archive diff --git a/env.example b/env.example index 68be8d6d249..9a4acb0dc51 100755 --- a/env.example +++ b/env.example @@ -111,9 +111,6 @@ FORBID_REDIRECTION=false GITLAB_SERVER_COLLECT_ALL_USERS=true ########################## -# In plugin gitextractor, use go-git to collector repo's data -########################## -USE_GO_GIT_IN_GIT_EXTRACTOR=false # NOTE that COMMIT_FILES is part of the COMMIT_STAT SKIP_COMMIT_STAT=false SKIP_COMMIT_FILES=true